From bc35583b03beedc9e53baeecd887366995473036 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 04:34:38 +0000 Subject: [PATCH 001/181] loom cell attach turns a prepared first cell into a serving one; prepare takes the same lock The attach command holds the attach lock on a direct, marked session of the cell's Postgres for its run and resumes by the directory row and the cell's mode. It refuses before any write on the stores' identity, the session, the roles, settings and subscriptions that would let an unchecked writer through, the database encoding, durability, its own grants, the ClickHouse writers, the table list and the processes' heartbeats; posts attach/begin; creates the triggers under the migration lock; in recording scans and writes the tenancy rows, refusing an org past ORG_MAX_BYTES; commits enforced, proves it durable, waits out every transaction that could have read recording, rescans, places every org in batches and marks the cell serving. A refusal after the mode commit leaves the cell enforced and attaching, and a rerun resumes there. Prepare refuses a session behind a pooler or a database that is not UTF8, and takes the attach lock. Doctor runs attach's Postgres reads again and reads stray orgs with --stray-orgs. instances.read_recent keeps stopped rows inside the widened window, and PRUNE_AFTER_S names the sweep's hour. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell.py | 4 +- server/cli/cell_attach.py | 756 +++++++++++++++++++ server/cli/cell_attach_beats.py | 324 ++++++++ server/cli/cell_attach_pg.py | 690 +++++++++++++++++ server/cli/cell_prepare.py | 42 +- server/cli/cell_scan.py | 50 +- server/cli/doctor.py | 47 +- server/cli/doctor_cell.py | 166 +++- server/core/cell_triggers.py | 139 ++-- server/core/instances.py | 57 +- tests/integration/test_cell_attach.py | 635 ++++++++++++++++ tests/integration/test_cell_scan.py | 2 +- tests/unit/test_cell_attach.py | 682 +++++++++++++++++ tests/unit/test_cell_org_triggers.py | 2 +- tests/unit/test_cell_prepare.py | 3 + tests/unit/test_doctor_client_checks.py | 2 +- tests/unit/test_doctor_client_client_only.py | 2 +- 17 files changed, 3529 insertions(+), 74 deletions(-) create mode 100644 server/cli/cell_attach.py create mode 100644 server/cli/cell_attach_beats.py create mode 100644 server/cli/cell_attach_pg.py create mode 100644 tests/integration/test_cell_attach.py create mode 100644 tests/unit/test_cell_attach.py diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..65f18990d8 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,7 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is -`server/cli/cell_prepare.py`. +`server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; `credential --cell --next | --finish | --revert` rotates a cell's. @@ -459,6 +459,8 @@ def _add_cell_parser(sub: Any) -> None: remove.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") remove.set_defaults(func=_cmd_remove) + from server.cli.cell_attach import add_attach_parser from server.cli.cell_prepare import add_prepare_parser add_prepare_parser(csub) + add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py new file mode 100644 index 0000000000..368f74ebf8 --- /dev/null +++ b/server/cli/cell_attach.py @@ -0,0 +1,756 @@ +"""`loom cell attach`: turn a prepared first cell into a `serving` one +(docs/architecture/cells.md, "Attaching a cell"; the runbook is +docs/operations/cells.md). + +A run holds the attach lock on a direct session of the cell's Postgres for its life, and +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: + +1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +2. the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, + the table list, durability, attach's own grants; +3. `attach/begin` (the router probes the stores and writes the fingerprints); +4. the mode: `recording`, or `enforced` on a resume; +5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; +7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode + only rises, so every later refusal leaves the cell `enforced` and `attaching`; +8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction + that could have read `recording`, the rescan, the tenancy rows and their placements + (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals +and the scan and prints what would be placed, writing nothing. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from collections.abc import Callable, Iterable +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli import cell_scan +from server.cli._ch import writes +from server.cli.cell_ch import Refusal, Refused, read_topology +from server.core import cell, cell_triggers +from server.settings import settings + +_WAIT_S = 600 # not a setting: a flag's default +_SCAN_BUDGET_S = 7200 # not a setting: a flag's default +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on the lock session's catalog reads +_HEALTH_TIMEOUT = httpx.Timeout(10.0, connect=5.0) # not a setting: one health read + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return f"{args.cell!r} is no cell name (1 to 32 of a-z, 0-9, -)" + if args.cell != settings.cell: + return f"LOOM_CELL is {settings.cell!r}: attach runs with LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return ( + "a cell keeps its state in Postgres: set LOOM_STORAGE_AUTHORITY=postgres and " + "LOOM_PG_DSN" + ) + if args.wait < 0 or args.scan_budget <= 0: + return "--wait is 0 or more seconds, --scan-budget more than 0" + return "" + + +class Router: + """The router's operator API for one cell.""" + + def __init__( + self, url: httpx.URL, name: str, transport: httpx.AsyncBaseTransport | None + ) -> None: + self.url, self.name, self.transport = url, name, transport + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + timeout=cell_cli._OPERATOR_TIMEOUT, + trust_env=self.url.scheme == "https", + transport=self.transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" + url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) + try: + async with self._client() as client: + r = await (client.get(url) if body is None else client.post(url, json=body)) + except httpx.HTTPError as e: + raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None + answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused("router", cell_cli.not_the_router(r)) + if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") + if r.status_code != 200: + raise _refused( + "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" + ) + return answer + + async def health(self, host: str) -> str: + """The deployment id `https:///v1/health/deployment` answers ('' when none).""" + try: + async with httpx.AsyncClient( + timeout=_HEALTH_TIMEOUT, transport=self.transport + ) as client: + r = await client.get(f"https://{host}/v1/health/deployment") + answer = r.json() + except (httpx.HTTPError, ValueError): + return "" + return str(answer.get("deployment_id") or "") if isinstance(answer, dict) else "" + + +def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: + """Step 2's refusal of every org past `cell.ORG_MAX_BYTES`, before any tenancy row.""" + return [ + Refusal( + f"org of {len(org.encode())} bytes ({org[:64]}…)", + f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " + "remove it first (docs/operations/cells.md, the over-long org procedure)", + ) + for org, sources in sorted(found.too_long.items()) + ] + + +def _refused(subject: str, reason: str) -> Refused: + return Refused([Refusal(subject, reason)]) + + +def _encoded_size(body: Any) -> int: + return len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + + +def batches(orgs: Iterable[str], deployment_id: str) -> list[list[str]]: + """`orgs` in `attach/place` batches: at most ATTACH_BATCH each, and an encoded body + within the router's body bound.""" + from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES + + out: list[list[str]] = [] + batch: list[str] = [] + for org in sorted(orgs): + if batch and ( + len(batch) >= ATTACH_BATCH + or _encoded_size({"deployment_id": deployment_id, "orgs": [*batch, org]}) + > MAX_BODY_BYTES + ): + out.append(batch) + batch = [] + batch.append(org) + if batch: + out.append(batch) + return out + + +class Run: + """One attach run: its sessions, what it read, and what it prints.""" + + def __init__( + self, + ch: Any, + args: argparse.Namespace, + router: Router, + log: Callable[[str], None], + ) -> None: + self.ch, self.args, self.router, self.log = ch, args, router, log + self.name = args.cell + self.wait = apg.Wait(args.wait) + self.lock_conn: Any = None + self._lock_cm: Any = None + self.row: dict[str, Any] = {} + self.mode = "" + self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) + self.keys_fp = "" + self.report: dict[str, Any] = {"cell": self.name} + + # -- the lock session -------------------------------------------------------------- + + async def open_lock(self) -> None: + from server.core import pg + + self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self.lock_conn = await self._lock_cm.__aenter__() + if refusals := await apg.direct_refusals(self.lock_conn, "attach"): + raise Refused(refusals) + await apg.take_lock(self.lock_conn, "attach") + + async def close_lock(self) -> None: + if self._lock_cm is not None: + await self._lock_cm.__aexit__(None, None, None) + self._lock_cm = self.lock_conn = None + + async def confirm_lock(self) -> None: + """The lock is held still, or taken again (refusing when another run holds it now), + and the cell's state read again: a cleanup, a prepare rerun or another attach could + have run in the gap.""" + try: + if await apg.lock_held(self.lock_conn): + return + except Exception as e: # noqa: BLE001 — a lost session is taken again below + self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") + else: + self.log("the attach lock was lost; taking it again") + try: + await self.close_lock() + except Exception: # noqa: BLE001 — the old session is gone either way + self._lock_cm = self.lock_conn = None + await self.open_lock() + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): + raise _refused( + "cell state", + f"changed while the lock was lost (now {state}, first row {first}): rerun attach", + ) + row = await self.router.call("") + if ( + row.get("deployment_id") != self.row.get("deployment_id") + or row.get("state") != "attaching" + ): + raise _refused("directory row", "changed while the lock was lost: rerun attach") + + # -- the refusals ------------------------------------------------------------------ + + async def read_own_row(self) -> None: + own = await self.lock_conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("postgres loom.cell_self", "absent: run loom cell prepare first") + self.own_fp = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + + def judge_beats(self, stopped_unset: str) -> Callable[[list[beats.Beat]], beats.Verdict]: + def _judge(found: list[beats.Beat]) -> beats.Verdict: + if stopped_unset == "ignore": + found = [b for b in found if b.live or b.cell == self.name] + return beats.judge( + found, self.name, self.own_fp, "wait" if stopped_unset == "wait" else "refuse" + ) + + return _judge + + async def check_beats(self, stopped_unset: str) -> beats.Verdict: + verdict = await beats.settle( + lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait + ) + (self.keys_fp,) = verdict.keys_fps + return verdict + + async def stores_refusals(self) -> list[Refusal]: + from server.core import pg_profile + + out = [] + row = self.row + if not row.get("is_first") or row.get("state") != "attaching": + out.append( + Refusal( + "directory row", + f"is {'the first' if row.get('is_first') else 'a later'} cell, " + f"{row.get('state')}: this attach is the first cell's, while attaching", + ) + ) + if row.get("other"): + out.append( + Refusal( + "directory row", + f"another cell, {row['other']}, is registered: remove it first " + "(loom cell remove)", + ) + ) + expected = str(row.get("deployment_id") or "") + recorded_pg = await self.lock_conn.fetchval("SELECT deployment_id FROM loom.authority") + if recorded_pg != expected: + out.append( + Refusal( + "postgres", f"records deployment id {recorded_pg!r}, the directory {expected!r}" + ) + ) + recorded = await pg_profile.recorded_authority(self.ch) + if recorded is None or str(recorded["deployment_id"]) != expected: + out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name): + out.append( + Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") + ) + if state not in ("recording", "enforced"): + out.append(Refusal("postgres cell state", f"reads {state}")) + answered = await self.router.health(str(row.get("host") or "")) + if answered != expected: + out.append( + Refusal( + f"https://{row.get('host')}/v1/health/deployment", + f"answers deployment id {answered!r}, not {expected!r}", + ) + ) + return out + + async def durability_refusals(self) -> list[Refusal]: + from server.core import pg, pg_profile + + out = [] + verdict = pg_profile.durability_verdict("synchronous", *await pg_profile.standby_state()) + if verdict.refusal: + out.append(Refusal("postgres durability", verdict.refusal)) + try: + await pg.check_durable_message() + except Exception as e: # noqa: BLE001 — a role that may not write it refuses + out.append(Refusal("postgres durability", f"pg_logical_emit_message refused: {e}")) + if (now := await pg_profile.standby_now()) != "synchronous": + out.append( + Refusal("postgres standbys", f"commits are acknowledged as {now}, not synchronous") + ) + return out + + async def ch_limit_refusals(self) -> list[Refusal]: + """A profile or quota that caps the rows or bytes attach's user reads would end a + scan early.""" + row = await self.ch.query_one( + "SELECT getSetting('max_rows_to_read') AS r, getSetting('max_bytes_to_read') AS b" + ) + out = [] + if row and (int(row["r"]) or int(row["b"])): + out.append( + Refusal("clickhouse user", "a settings profile caps the rows or bytes it reads") + ) + quota = await self.ch.query_all( + "SELECT quota_name FROM system.quota_usage " + "WHERE max_read_rows > 0 OR max_read_bytes > 0" + ) + out += [ + Refusal(f"clickhouse quota {q['quota_name']}", "caps the rows or bytes attach reads") + for q in quota + ] + return out + + async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: + from server.cli import cell_writers + + topology = await read_topology(self.ch) + if refusals := topology.refusals(): + return refusals, topology, {} + names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + allowed = cell_writers.Allowed( + users=frozenset(self.args.ch_allow_user), + user_directories=frozenset(self.args.ch_allow_user_directory), + objects=frozenset(self.args.ch_allow_object), + replicas=frozenset(self.args.ch_allow_replica), + ) + return await cell_writers.check_writers(self.ch, topology, names, allowed), topology, names + + async def catalog_refusals(self) -> list[Refusal]: + """The Postgres checks that run again before the mode commit and in step 4.""" + holders = await apg.privilege_holders(self.lock_conn) + self.report["own_roles"] = holders.own + out = apg.privilege_refusals(holders, self.args.pg_allow_role) + out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) + return out + + async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: + """Every refusal before any write, named together.""" + out = await self.stores_refusals() + out += await apg.encoding_refusals(self.lock_conn) + out += await self.catalog_refusals() + out += await self.durability_refusals() + rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) + tables_pg = [f"loom.{r['relname']}" for r in rows] + out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await self.ch_limit_refusals() + if out: + raise Refused(out) + state, _ = await apg.read_mode(self.lock_conn) + # A stopped process with LOOM_CELL unset refuses while the cell can still be + # backed out; on a resume in `enforced` step 4 waits for it to age out. + await self.check_beats("refuse" if state == "recording" else "ignore") + writers, topology, names = await self.writer_refusals() + out += writers + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + out += listed.refusals + if out: + raise Refused(out) + return topology, names, listed + + # -- begin ------------------------------------------------------------------------- + + async def begin(self) -> None: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + from server.router.limits import ATTACH_BEGIN_MAX_BYTES + + current, nxt = self.own_fp + nonces = _new_nonces() + body = { + "deployment_id": self.row["deployment_id"], + "current_fp": current, + "next_fp": nxt, + "keys_fp": self.keys_fp, + "ch_allow_users": sorted(self.args.ch_allow_user), + "ch_allow_user_directories": sorted(self.args.ch_allow_user_directory), + "ch_allow_objects": sorted(self.args.ch_allow_object), + "ch_allow_replicas": sorted(self.args.ch_allow_replica), + "pg_allow_roles": sorted(self.args.pg_allow_role), + "nonces": nonces, + } + if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: + raise _refused( + "attach/begin", + f"its body is {size} bytes, past the router's {ATTACH_BEGIN_MAX_BYTES}: the " + "fingerprints or the allow lists are too large", + ) + try: + await _write_nonces(self.ch, nonces) + self.row = {**self.row, **await self.router.call("/attach/begin", body)} + finally: + await _delete_nonces(self.ch, nonces) + + # -- the scans --------------------------------------------------------------------- + + async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: + budget = cell_scan.Budget(self.args.scan_budget) + current, nxt = self.own_fp + return await cell_scan.scan( + self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) + ) + + def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: + return { + "malformed": {k: v[:20] for k, v in sorted(found.malformed.items())}, + "too_long": { + f"{o[:64]}… ({len(o.encode())} bytes)": sorted(s) for o, s in found.too_long.items() + }, + } + + async def triggers(self) -> None: + if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + raise Refused(refusals) + + # -- step 3 ------------------------------------------------------------------------ + + async def ready_for_commit(self) -> bool: + """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, + a booting process skips the try.""" + verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) + if verdict.refusals: + raise Refused(verdict.refusals) + return not verdict.booting + + async def commit_mode(self) -> None: + from server.core import pg + + await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + try: + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + ) + except Refused: + raise + except Exception as e: # noqa: BLE001 — the durable read says whether it landed + self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.end_mode_transactions(conn, self.confirm_lock) + + async def durable(self) -> int: + """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + xmax, above the read's own xid.""" + from server.core import pg_profile + + while True: + try: + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read() + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") + break + except Refused: + raise + except Exception as e: + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused("postgres durable read", "did not commit within --wait") from e + if read.first != (True, self.name) or read.mode != "enforced": + raise _refused( + "postgres cell state", + f"the durable read saw mode {read.mode!r} and first row {read.first}: the mode " + "commit did not land; rerun attach", + ) + self.mode = "enforced" + xmax = await apg.snapshot_xmax(self.lock_conn) + if xmax <= read.xid: + raise _refused("postgres snapshot", f"its xmax {xmax} is not above {read.xid}") + return xmax + + # -- step 4 ------------------------------------------------------------------------ + + async def cheap_checks(self, stopped_unset: str) -> None: + from server.core import pg_profile + + out, _, _ = await self.writer_refusals() + out += [ + Refusal("postgres triggers", d.text) + for d in await cell_triggers.missing(self.lock_conn) + ] + out += await self.catalog_refusals() + if out: + raise Refused(out) + await self.check_beats(stopped_unset) + if (now := await pg_profile.standby_now()) != "synchronous": + raise _refused("postgres standbys", f"commits are acknowledged as {now}") + + async def final_checks(self, watch: beats.Watch) -> None: + from server.core import instances + + floor = apg.Wait(max(self.wait.left(), instances.boot_allowance_s())) + + async def _read() -> list[beats.Beat]: + found = await watch.read_once() + if refusals := watch.refusals(): + raise Refused(refusals) + # A cell-correct process whose cell row alone was swept. + return [b for b in found if not (b.cell is None and b.instance_id in watch.cell_ids)] + + verdict = await beats.settle( + _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() + ) + if verdict.keys_fps != {self.keys_fp}: + raise _refused( + "processes", + "the root-secret fingerprint changed during attach (a root-secret rotation): " + "rerun attach", + ) + + async def step4(self, xmax: int, topology: Any, names: Any) -> None: + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.wait_transactions(conn, xmax, self.wait) + await self.cheap_checks("wait") + since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) + watch = beats.Watch(self.ch, since, self.name) + if refusals := watch.refusals_up_front(): + raise Refused(refusals) + task = asyncio.create_task(watch.run()) + try: + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + found = await self.scan(topology, listed) + orgs = set(found.orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + placed = 0 + for batch in batches(orgs, self.row["deployment_id"]): + await self.confirm_lock() + answer = await self.router.call( + "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} + ) + placed += int(answer.get("placed") or 0) + self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) + writers, _, _ = await self.writer_refusals() + if writers: + raise Refused(writers) + finally: + task.cancel() + await self.final_checks(watch) + row = await self.router.call("") + current, nxt = self.own_fp + if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + raise _refused( + "postgres loom.cell_self", + "differs from the directory's fingerprints (a loom cell update between them?)", + ) + state, _ = await apg.read_mode(self.lock_conn) + if state != "enforced": + raise _refused("postgres cell state", f"reads {state}, not enforced") + + async def serving(self) -> dict[str, Any]: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + + nonces = _new_nonces() + try: + await _write_nonces(self.ch, nonces) + return await self.router.call( + "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} + ) + finally: + await _delete_nonces(self.ch, nonces) + + # -- the run ----------------------------------------------------------------------- + + async def done(self) -> bool: + """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + state, _ = await apg.read_mode(conn) + if state != "enforced": + raise _refused( + "postgres cell state", + f"the directory says serving and the cell reads {state}: run loom cell restore", + ) + return True + + async def run(self) -> int: + self.row = await self.router.call("") + if self.row.get("state") == "serving": + await self.done() + self.report.update(state="serving", done=True) + return 0 + await self.open_lock() + try: + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") + await self.triggers() + await self.precommit(found) + await self.commit_mode() + else: + await self.triggers() + await self.router.call( + "/attach/enforced", {"deployment_id": self.row["deployment_id"]} + ) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 + finally: + await self.close_lock() + + async def precommit(self, found: cell_scan.ScanResult) -> None: + from server.core import pg_profile + + out = await self.catalog_refusals() + if (now := await pg_profile.standby_now()) != "synchronous": + out.append(Refusal("postgres standbys", f"commits are acknowledged as {now}")) + if found.elapsed_s > self.args.scan_budget / 2: + out.append( + Refusal( + "--scan-budget", + f"step 2 took {found.elapsed_s:.0f} s, more than half of " + f"{self.args.scan_budget} s, and step 4 reads the same tables again: " + "raise it", + ) + ) + if out: + raise Refused(out) + verdict = await self.check_beats("refuse") + if refusals := beats.final_refusals(verdict): + raise Refused(refusals) + + async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: + found = await self.scan(topology, listed) + per_source: dict[str, list[str]] = {} + for org, sources in found.orgs.items(): + for s in sources: + per_source.setdefault(s, []).append(org) + self.report.update( + dry_run=True, + scan_seconds=round(found.elapsed_s, 1), + half_budget_ok=found.elapsed_s <= self.args.scan_budget / 2, + orgs=len(found.orgs), + per_source={ + s: {"count": len(o), "sample": sorted(o)[:5]} for s, o in sorted(per_source.items()) + }, + skipped=self._named(found), + ) + return 0 + + +async def attach( + ch: Any, + args: argparse.Namespace, + url: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> int: + def log(text: str) -> None: + print(f"loom cell attach: {text}", file=sys.stderr) + + run = Run(ch, args, Router(url, args.cell, transport), log) + started = time.monotonic() + try: + code = await run.run() + except Refused as e: + for r in e.refusals: + log(f"refusing: {r}") + mode = ( + "enforced, attaching: rerun attach to resume" + if run.mode == "enforced" + else ("unchanged" if not run.mode else f"{run.mode}") + ) + log(f"the cell stays {mode}") + return 1 + run.report["seconds"] = round(time.monotonic() - started, 1) + print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) + return code + + +def cmd_attach(args: argparse.Namespace) -> int: + if problem := _args_problem(args): + print(f"loom cell attach: {problem}", file=sys.stderr) + return 2 + url = cell_cli.operator_router(args) + if isinstance(url, str): + print(f"loom cell attach: {url}", file=sys.stderr) + return 2 + from server.cli._ch import run_connected + from server.core.ch import statement_timeout + + async def _run(ch: Any) -> int: + try: + return await attach(ch, args, url) + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + print( + f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", + file=sys.stderr, + ) + return 1 + + # Every ClickHouse statement, the connect included, may run as long as a scan may. + with statement_timeout(args.scan_budget): + return run_connected(_run, org_free=writes("attaches the cell; places the orgs it holds")) + + +def add_attach_parser(csub: Any) -> None: + p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Run the refusals and the scan and print what would be placed; write nothing " + "(the router's own begin refusals do not run)", + ) + for flag, what in ( + ("--ch-allow-user", "A ClickHouse user that may write the configured databases"), + ("--ch-allow-user-directory", "A ClickHouse user directory the check cannot read"), + ("--ch-allow-object", "A ClickHouse object (db.name) that writes the databases"), + ("--ch-allow-replica", "A replica outside the cluster (db.table:replica)"), + ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), + ): + p.add_argument(flag, action="append", default=[], help=what) + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py new file mode 100644 index 0000000000..a2fc37701d --- /dev/null +++ b/server/cli/cell_attach_beats.py @@ -0,0 +1,324 @@ +"""The first attach's judgment of the cell's processes, from their heartbeats +(docs/architecture/cells.md, "Attaching a cell" and "The heartbeat's keys"). + +Every check reads both heartbeat tables across every deployment sharing the ClickHouse, as +prepare does, stopped rows included (`instances.read_recent`), and judges in one order: + +- `cell` first: a live process whose `LOOM_CELL` is unset or another cell refuses at once; + a stopped one, or a row with no cell row that is no longer live, may still be writing + for the widened window after its stop beat, and is refused, or waited on until it ages + out, as the caller's `stopped_unset` says; +- then the keys a current build reports (`serves`, `marked`, `router`): a cell row + without them refuses, even while booting; +- then booting: a live instance with no cell row yet, no tenancy mode yet, `marked` null, + or `router` null where `serves` is true, is waited on; `marked` false, or `router` false + where `serves` is true, refuses, naming the process and the router's cause; +- and the fingerprint: each live process's matches `loom.cell_self`'s current or next. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from server.cli.cell_attach_pg import Wait +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, instances + +StoppedUnset = Literal["refuse", "wait"] + + +@dataclass(frozen=True) +class Beat: + """One process: its instance row (`live`: inside the live window and not stopped) and + its cell row (`cell` None when it wrote none), the fingerprint JSON decoded.""" + + instance_id: str + live: bool + stopped: bool + keys_fp: str + last_beat: str + cell: str | None = None + mode: str = "" + value: dict[str, Any] | None = None + + +async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: + rows = await instances.read_recent(ch, since=since) + ids = [str(r["instance_id"]) for r in rows] + cells = await ch.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": ids}, + ) + by_id = {str(r["instance_id"]): r for r in cells} + out = [] + for r in rows: + found = by_id.get(str(r["instance_id"])) + beat = Beat( + instance_id=str(r["instance_id"]), + live=bool(r["live"]), + stopped=bool(r["stopped"]), + keys_fp=str(r["keys_fp"]), + last_beat=str(r["last_beat"]), + cell=None if found is None else str(found["cell"]), + mode="" if found is None else str(found["tenancy_mode"]), + value=None if found is None else _decoded(str(found["fingerprint"])), + ) + out.append(beat) + return out + + +def _decoded(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except ValueError: + return None + return value if isinstance(value, dict) else None + + +@dataclass +class Verdict: + """What one read found: the refusals, the processes waited on (booting, or a stopped + process with `LOOM_CELL` unset aging out), the live processes judged, their `keys_fp` + values, and whether one reports SCIM on.""" + + refusals: list[Refusal] = field(default_factory=list) + booting: list[str] = field(default_factory=list) + aging: list[str] = field(default_factory=list) + live: int = 0 + keys_fps: set[str] = field(default_factory=set) + scim: bool = False + + +def judge( + beats: Iterable[Beat], + name: str, + accepted: Iterable[Mapping[str, Any] | None], + stopped_unset: StoppedUnset, +) -> Verdict: + """The verdict on `beats` for the cell `name`, whose processes run `accepted` (its + `loom.cell_self` current and next).""" + values = [v for v in accepted if v is not None] + out = Verdict() + for b in beats: + who = f"process {b.instance_id}" + if b.cell is None: + if b.live: + out.booting.append(b.instance_id) + else: + _stopped(out, b, "stopped before reporting its LOOM_CELL", stopped_unset) + continue + if b.cell != name: + runs = f"runs with LOOM_CELL={b.cell}" if b.cell else "runs with LOOM_CELL unset" + if b.live: + out.refusals.append(Refusal(who, f"{runs}: set LOOM_CELL={name} on it")) + else: + _stopped(out, b, f"{runs} and stopped", stopped_unset) + continue + if not b.live: + continue + value = b.value or {} + if not all(k in value for k in ("serves", "marked", "router")): + out.refusals.append( + Refusal(who, "reports no serves, marked or router key: upgrade it to this release") + ) + continue + if value["marked"] is False: + out.refusals.append( + Refusal( + who, + "its Postgres sessions lose the tenancy marker (a pooler?): point " + "LOOM_PG_DSN straight at Postgres", + ) + ) + continue + serves = value["serves"] is True + if serves and value["router"] is False: + why = value.get("router_why") or "no cause reported" + out.refusals.append(Refusal(who, f"cannot read its router: {why}")) + continue + if value["marked"] is None or (serves and value["router"] is None) or not b.mode: + out.booting.append(b.instance_id) + continue + fields = value.get("fields") + if not isinstance(fields, dict) or not any(cell.matches(fields, v) for v in values): + out.refusals.append( + Refusal(who, "runs neither loom.cell_self's current nor its next configuration") + ) + continue + out.live += 1 + out.keys_fps.add(b.keys_fp) + out.scim = out.scim or bool(fields.get("scim")) + return out + + +def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> None: + if stopped_unset == "wait": + out.aging.append(b.instance_id) + return + out.refusals.append( + Refusal( + f"process {b.instance_id}", + f"{what} at {b.last_beat}, and may still be writing (its drain and retirement " + "after its stop beat)", + ) + ) + + +def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: + out = [] + if not verdict.live and not verdict.booting: + out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) + if len(verdict.keys_fps) > 1: + out.append( + Refusal( + "processes", + "the live processes report different root-secret fingerprints (keys_fp): " + "finish the root-secret rollout first", + ) + ) + if verdict.scim and not scim_ok: + out.append( + Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") + ) + return out + + +async def settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + *, + extend_s: float = 0.0, +) -> Verdict: + """Read and judge until nothing is waited on: Refused at the first refusal, and once + `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended + once, to `extend_s` past the first sight of the last process first seen booting.""" + first_seen: dict[str, float] = {} + extended = False + while True: + verdict = judge_beats(await read()) + if verdict.refusals: + raise Refused(verdict.refusals) + if not verdict.booting and not verdict.aging: + if refusals := final_refusals(verdict): + raise Refused(refusals) + return verdict + for i in verdict.booting: + first_seen.setdefault(i, wait.spent) + if await wait.pause(): + continue + if extend_s and not extended and verdict.booting: + extended = True + wait.seconds = max(wait.seconds, max(first_seen[i] for i in verdict.booting) + extend_s) + if await wait.pause(): + continue + raise Refused( + [Refusal(f"process {i}", "still booting at the end of --wait") for i in verdict.booting] + + [ + Refusal(f"process {i}", "stopped with LOOM_CELL unset, not aged out within --wait") + for i in verdict.aging + ] + ) + + +# -- the rescan's watch ---------------------------------------------------------------- + +_READ_TIMEOUT_S = 30.0 # not a setting: one heartbeat read's bound +_SKEW_S = 10.0 # not a setting: a margin for clock skew between ClickHouse hosts + + +def watch_period_s() -> float: + """`T'`: how long a row whose window reaches past the rescan's start is kept before the + leader's sweep may drop it, less a margin for clock skew.""" + return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + + +class Watch: + """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's + start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at + the end even once the leader swept its rows.""" + + def __init__( + self, ch: Any, since: str, name: str, clock: Callable[[], float] = time.monotonic + ) -> None: + self.ch, self.since, self.name = ch, since, name + self._clock = clock + self.cell_ids: set[str] = set() + self.unset: dict[str, str] = {} + self.no_cell: dict[str, str] = {} + self.gap = False + self._good = clock() + + def refusals_up_front(self) -> list[Refusal]: + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + async def read_once(self) -> list[Beat]: + try: + async with asyncio.timeout(_READ_TIMEOUT_S): + beats = await read_beats(self.ch, since=self.since) + except Exception: + self._check_gap() + raise + self._check_gap() + self._good = self._clock() + for b in beats: + if b.cell == self.name: + self.cell_ids.add(b.instance_id) + elif b.cell is not None: + self.unset[b.instance_id] = b.last_beat + elif b.stopped: + self.no_cell[b.instance_id] = b.last_beat + return beats + + def _check_gap(self) -> None: + if self._clock() - self._good > watch_period_s(): + self.gap = True + + async def run(self) -> None: + """Read until cancelled; a failed read is retried at the next period.""" + while True: + await asyncio.sleep(watch_period_s() / 2) + try: + await self.read_once() + except Exception: # noqa: BLE001 — a failed read is retried; a gap refuses + continue + + def refusals(self) -> list[Refusal]: + out = [ + Refusal( + f"process {i}", + f"ran with LOOM_CELL unset or another cell during the rescan (last beat {at})", + ) + for i, at in sorted(self.unset.items()) + ] + out += [ + Refusal( + f"process {i}", + f"stopped during the rescan before reporting its LOOM_CELL (last beat {at})", + ) + for i, at in sorted(self.no_cell.items()) + if i not in self.cell_ids + ] + if self.gap: + out.append( + Refusal( + "heartbeats", + "not read for longer than the leader keeps a stopped process's rows: " + "rerun attach", + ) + ) + return out diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py new file mode 100644 index 0000000000..6ee4432907 --- /dev/null +++ b/server/cli/cell_attach_pg.py @@ -0,0 +1,690 @@ +"""The first attach's reads and writes on the cell's Postgres (docs/architecture/cells.md, +"Attaching a cell"): the session checks, the one-run lock, the catalog checks attach and +`loom doctor` share (the roles that can make an org held or lower the mode, the settings +and subscriptions that pass an unchecked writer), the trigger pass, the mode commit, its +durable read and the wait for the transactions that could have read `recording`. + +Every check returns its refusals as `cell_ch.Refusal` (subject, reason), as the org scan +and the ClickHouse writer check do, so attach and doctor print the same names.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, cell_triggers, pg +from server.core.cell import ATTACH_LOCK_KEY + +_sleep = asyncio.sleep # the seam the wait tests replace + +# The pause between polls of a wait. +POLL_S = 1.0 # not a setting: a poll interval +# The migration lock's wait a trigger try (a boot holds it for its migrations). +_MIGRATION_LOCK_TIMEOUT = "10s" +# The mode commit's lock wait a try, and the statement bound above it. +_MODE_LOCK_TIMEOUT = "2s" +_MODE_STATEMENT_TIMEOUT = "5s" +# The server ends the lock session soon after its client is gone, releasing the lock. +_KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} + + +class Wait: + """`--wait`: the seconds attach's waits share, counting only the time spent waiting + (never scan time).""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self.spent = 0.0 + self._clock = clock + + def left(self) -> float: + return max(0.0, self.seconds - self.spent) + + def now(self) -> float: + return self._clock() + + async def pause(self, seconds: float = POLL_S) -> bool: + """Wait `seconds` (at most what is left) and count them; False once none is left.""" + step = min(seconds, self.left()) + if step <= 0: + return False + start = self._clock() + await _sleep(step) + self.spent += max(step, self._clock() - start) + return True + + +def _split(key: int) -> str: + """The `pg_locks` columns that hold a bigint advisory key, in this database.""" + return ( + f"l.locktype = 'advisory' AND l.classid = ({key} >> 32)::oid " + f"AND l.objid = ({key} & 4294967295)::oid AND l.objsubid = 1 " + "AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ) + + +_LOCK_HOLDER = f""" +SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted +""" + +_IDENTITY = ( + "SELECT pg_backend_pid() AS pid, backend_start FROM pg_stat_activity " + "WHERE pid = pg_backend_pid()" +) + + +async def direct_refusals(conn: Any, what: str) -> list[Refusal]: + """Whether `conn` is one direct, marked session: the same backend across two + transactions, and the tenancy marker kept through `RESET ALL` (a pooler that strips + startup parameters drops it). Behind a transaction pooler a session-level lock would + outlive the CLI on the pooler's server connection, and attach's tenancy rows would be + refused as an unchecked writer's.""" + async with conn.transaction(): + first = await conn.fetchrow(_IDENTITY) + async with conn.transaction(): + second = await conn.fetchrow(_IDENTITY) + out = [] + if first is None or second is None or tuple(first) != tuple(second): + out.append( + Refusal( + "postgres session", + f"{what}'s session moved between backends: point LOOM_PG_DSN straight at " + "Postgres, never through a pooler", + ) + ) + await conn.execute("RESET ALL") # a startup value survives it, a SET does not + if await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) != "on": + out.append( + Refusal( + "postgres session", + f"{what}'s session loses {pg.TENANCY_MARKER}: run it with LOOM_CELL set, " + "LOOM_PG_DSN straight at Postgres", + ) + ) + return out + + +_HELD = f""" +SELECT EXISTS (SELECT 1 FROM pg_locks l + WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted AND l.pid = pg_backend_pid()) +""" + + +async def lock_holder(conn: Any) -> str: + """Who holds the attach lock ('' when no one), never a query's text.""" + rows = await conn.fetch(_LOCK_HOLDER) + return "; ".join( + f"pid {r['pid']} ({r['application_name'] or 'no name'}, from {r['client'] or 'local'}, " + f"since {r['backend_start']})" + for r in rows + ) + + +async def take_lock(conn: Any, what: str) -> None: + """Take the attach lock on `conn` for the session's life, confirmed held by this + backend; Refused naming its holder when another run holds it.""" + for name, value in _KEEPALIVE.items(): + await conn.execute(f"SET {name} = {value}") + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", ATTACH_LOCK_KEY): + holder = await lock_holder(conn) or "a session that just ended" + raise Refused( + [ + Refusal( + "attach lock", + f"held by {holder}: an attach, prepare or update is running; {what} " + "runs one at a time", + ) + ] + ) + if not await lock_held(conn): + raise Refused( + [ + Refusal( + "attach lock", + "taken but not held by this session's backend: a pooler between " + "this command and Postgres; point LOOM_PG_DSN straight at it", + ) + ] + ) + + +async def lock_held(conn: Any) -> bool: + """Whether `conn`'s backend holds the attach lock.""" + return bool(await conn.fetchval(_HELD)) + + +async def encoding_refusals(conn: Any) -> list[Refusal]: + """A cell's database is UTF8: its `text` then holds no NUL or invalid UTF-8, so every + tenancy row within the length limit is an org a request can name.""" + encoding = await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()" + ) + if encoding == "UTF8": + return [] + return [ + Refusal( + "postgres database", + f"its encoding is {encoding}, not UTF8: a cell's database is UTF8 " + "(recreate it with ENCODING 'UTF8' and restore into it)", + ) + ] + + +# -- who can make an org held, or lower the mode -------------------------------------- + +_SYSTEM_SCHEMA = ( + "n.nspname NOT IN ('pg_catalog', 'information_schema') " + "AND n.nspname NOT LIKE 'pg\\_toast%' AND n.nspname NOT LIKE 'pg\\_temp\\_%'" +) + +# Every right a login role may hold that lets it make an org held, lower the mode, or run +# its own SQL as the servers on their marked sessions: each row a role and why. +_RIGHTS = f""" +WITH owned AS ( + SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + UNION ALL + SELECT p.proowner, 'owns loom.' || p.proname || '()' FROM pg_proc p + WHERE p.oid IN (to_regprocedure('loom.tenancy_refuse_unheld()'), + to_regprocedure('loom.tenancy_refuse_unmarked()'), + to_regprocedure('loom.cell_first_immutable()')) + UNION ALL + SELECT d.datdba, 'owns the database' FROM pg_database d + WHERE d.datname = current_database() + UNION ALL + SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} +), +login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) +SELECT r.rolname AS role, w.why FROM login r, LATERAL ( + SELECT 'may insert into loom.tenancy' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') + UNION ALL SELECT 'may update loom.tenancy (org)' + WHERE has_column_privilege(r.oid, 'loom.tenancy', 'org', 'UPDATE') + UNION ALL SELECT 'may update loom.tenancy_mode (mode)' + WHERE has_column_privilege(r.oid, 'loom.tenancy_mode', 'mode', 'UPDATE') + UNION ALL SELECT 'may insert into loom.tenancy_mode' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy_mode', 'INSERT') + UNION ALL SELECT 'holds TRIGGER on loom.' || c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND has_table_privilege(r.oid, c.oid, 'TRIGGER') + UNION ALL SELECT 'may create in the database' + WHERE has_database_privilege(r.oid, current_database(), 'CREATE') + UNION ALL SELECT 'may create in schema ' || n.nspname FROM pg_namespace n + WHERE {_SYSTEM_SCHEMA} AND has_schema_privilege(r.oid, n.oid, 'CREATE') + UNION ALL SELECT 'may set session_replication_role' + WHERE has_parameter_privilege(r.oid, 'session_replication_role', 'SET') + OR has_parameter_privilege(r.oid, 'session_replication_role', 'ALTER SYSTEM') + UNION ALL SELECT 'may ALTER SYSTEM SET {pg.TENANCY_MARKER}' + WHERE has_parameter_privilege(r.oid, '{pg.TENANCY_MARKER}', 'ALTER SYSTEM') + UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g + WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') + AND pg_has_role(r.oid, g.oid, 'MEMBER') + UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' + END + FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') +) w(why) +ORDER BY 1, 2 +""" + +# The roles attach counts as its own: superusers, and a login role that itself owns +# loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the +# triggers, or lower the mode, whatever else it holds). +_OWN_ROLES = """ +SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( + SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) +)) OR rolname = current_user +ORDER BY 1 +""" + + +@dataclass(frozen=True) +class Holders: + """The login roles holding a right `_RIGHTS` counts: the ones attach counts as its own + (`own`, named in its output) and every other with its reasons.""" + + own: list[str] + others: dict[str, list[str]] + + +async def privilege_holders(conn: Any) -> Holders: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + others: dict[str, list[str]] = {} + for r in await conn.fetch(_RIGHTS): + if r["role"] not in own: + others.setdefault(str(r["role"]), []).append(str(r["why"])) + return Holders(own, others) + + +def privilege_refusals(holders: Holders, allowed: Iterable[str]) -> list[Refusal]: + allow = set(allowed) + return [ + Refusal( + f"postgres role {role}", + f"{'; '.join(why)}: it can make an org held or lower the mode outside attach; " + "revoke it, or name the role with --pg-allow-role", + ) + for role, why in sorted(holders.others.items()) + if role not in allow + ] + + +# -- what passes an unchecked writer through the triggers ----------------------------- + +_ROLE_SETTINGS = f""" +SELECT coalesce(r.rolname, 'every role') AS role, coalesce(d.datname, 'every database') AS db, + cfg +FROM pg_db_role_setting s +LEFT JOIN pg_roles r ON r.oid = s.setrole +LEFT JOIN pg_database d ON d.oid = s.setdatabase, +LATERAL unnest(s.setconfig) cfg +WHERE s.setdatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database())) + AND lower(split_part(cfg, '=', 1)) IN ('{pg.TENANCY_MARKER}', 'session_replication_role') +ORDER BY 1, 2, 3 +""" + +_SUBSCRIPTIONS = """ +SELECT subname, subenabled FROM pg_subscription +WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY 1 +""" + + +async def unmarked_refusals(conn: Any, session_passes: bool) -> list[Refusal]: + """What lets a session that sends no marker pass the triggers: the session read + (`session_passes`, doctor's unmarked session), an `ALTER ROLE … SET` or + `ALTER DATABASE … SET` of the marker or `session_replication_role` for this database or + every one, and any logical-replication subscription into this database (its apply + worker runs as `replica`, firing no origin trigger).""" + out = [] + if session_passes: + out.append( + Refusal( + "postgres unmarked session", + "a session that sends no marker is marked, or runs as replica (a role, " + "database, ALTER SYSTEM or options setting): every unchecked writer passes", + ) + ) + for r in await conn.fetch(_ROLE_SETTINGS): + out.append( + Refusal( + f"postgres setting for {r['role']} in {r['db']}", + f"{r['cfg']}: an unchecked writer under it passes the triggers; reset it", + ) + ) + for r in await conn.fetch(_SUBSCRIPTIONS): + state = "enabled" if r["subenabled"] else "disabled" + out.append( + Refusal( + f"postgres subscription {r['subname']}", + f"{state}: its apply worker fires no origin trigger; drop it", + ) + ) + return out + + +async def session_passes() -> bool: + """Whether a session that sends no marker passes the triggers all the same: marked, + or in the replica role that fires no origin trigger.""" + async with pg.dedicated(statement_timeout_ms=5000, marked=False) as conn: + value = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) + role = await conn.fetchval("SELECT current_setting('session_replication_role')") + return bool(value == "on" or role == "replica") + + +# -- attach's own role ------------------------------------------------------------------ + +_OWN_GRANTS = """ +SELECT need, held FROM (VALUES + ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), + ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), + ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), + ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), + ('UPDATE on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'UPDATE')), + ('SELECT on loom.cell_self', has_table_privilege('loom.cell_self', 'SELECT')), + ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), + ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), + ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', + has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) +) v(need, held) +WHERE NOT held +""" + + +async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: + """What attach's own role lacks (the guarded tables' owner roles are `missing`'s + `owner` defects), and a server that allows prepared transactions.""" + role = await conn.fetchval("SELECT current_user") + lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] + for table in tables: + if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): + lacks.append(f"SELECT on {table}") + out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] + out += await prepared_refusals(conn) + return out + + +async def prepared_refusals(conn: Any) -> list[Refusal]: + setting = await conn.fetchval("SELECT current_setting('max_prepared_transactions')::int") + if setting: + return [ + Refusal( + "postgres max_prepared_transactions", + f"is {setting}: a prepared transaction outlives every session, so step 4 " + "could not wait for it; set it to 0", + ) + ] + return [] + + +# -- the triggers ----------------------------------------------------------------------- + +_MIGRATION_LOCK = "SELECT pg_advisory_lock($1)" +_MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" + + +async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: + """Give every planned table its triggers (`cell_triggers.ensure_table` with + `recording_first`), one try at a time under the migration lock, retrying a table whose + lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield + by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + deadlock_ms = int( + await conn.fetchval( + "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " + "FROM pg_settings WHERE name = 'deadlock_timeout'" + ) + ) + out: list[Refusal] = [] + for table in await cell_triggers.planned_tables(conn): + yielded = False + while True: + lock_timeout = cell_triggers.LOCK_TIMEOUT + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_timeout, yielded = f"{deadlock_ms + 500}ms", True + done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) + if done: + break + if not await wait.pause(): + out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) + break + log(f"loom.{table}: its lock is busy; retrying") + if out: + return out + return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] + + +async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: + await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") + try: + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) == "55P03": + return False # a boot migrating: the next try + raise + finally: + await conn.execute("RESET lock_timeout") + try: + return await cell_triggers.ensure_table( + conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + ) + finally: + await conn.execute(_MIGRATION_UNLOCK, key) + + +def _busy_refusal(table: str, holders: list[Any]) -> Refusal: + named = "; ".join( + f"pid {h['pid']} ({'holds' if h['granted'] else 'waiting'}, {h['backend_type']}, " + f"{h['application_name'] or 'no name'}, transaction since {h['xact_start']}" + + (", an anti-wraparound vacuum, which never yields" if h["wraparound"] else "") + + ")" + for h in holders + ) + return Refusal( + f"postgres loom.{table}", + f"its triggers' lock was not taken within --wait; held by {named or 'no one now'}", + ) + + +# -- step 3: the mode commit ------------------------------------------------------------ + +# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on +# loom.tenancy_mode in this database, which only attach takes. +_MODE_HOLDERS = """ +SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted +FROM pg_locks l +JOIN pg_stat_activity a ON a.pid = l.pid +LEFT JOIN pg_roles r ON r.oid = a.usesysid +WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass + AND l.mode = 'ExclusiveLock' + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.pid <> pg_backend_pid() +""" + +_TIMED_OUT = frozenset({"55P03", "57014"}) + + +async def commit_enforced( + conn: Any, + confirm_lock: Callable[[], Any], + ready: Callable[[], Any], + wait: Wait, + log: Callable[[str], None], +) -> None: + """Raise the mode to `enforced` on `conn` (a session of its own, never the lock + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, + within `wait`; `confirm_lock` before each try and each termination (it refuses when + another run holds the attach lock), and `ready` before each try (False skips the try + while a process boots). After a failed try the in-flight mode transaction, found by its + lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the + durable read's to say.""" + while True: + await confirm_lock() + if await ready(): + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", f"the mode update read {status}" + ) + ] + ) + return + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + log("the mode's lock is busy; retrying") + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads it " + "FOR SHARE; rerun attach", + ) + ] + ) + + +async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: + """End an earlier run's mode transaction still in flight and wait until its lock + clears; Refused when its role is a superuser and attach's is not.""" + for _ in range(30): + rows = await conn.fetch(_MODE_HOLDERS) + if not rows: + return + superuser = bool( + await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") + ) + for r in rows: + if r["rolsuper"] and not superuser: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"an earlier mode transaction ({r['application_name'] or 'no name'}, " + f"since {r['xact_start']}) runs as a superuser: a superuser must " + "end it (pg_terminate_backend)", + ) + ] + ) + await confirm_lock() + await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) + await _sleep(0.1) + + +# -- the durable read ------------------------------------------------------------------- + +_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried + + +@dataclass(frozen=True) +class Durable: + """What the durable read saw: the mode, the `first` row, and the xid its logical + message took.""" + + mode: str + first: tuple[bool, str] | None + xid: int + + +async def durable_read() -> Durable: + """The mode and the `first` row read in a transaction that emits a transactional + logical message, whose commit waits for the synchronous standbys: once it returns, the + mode read is on them. Refused when the message took no xid.""" + async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") + xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + +async def snapshot_xmax(conn: Any) -> int: + return int(await conn.fetchval("SELECT pg_snapshot_xmax(pg_current_snapshot())::text")) + + +# -- step 4: the transactions that could have read `recording` -------------------------- + +# Every backend but this statement's own: of the cell's database, one whose xid or (but +# an autovacuum's) snapshot is older than X; of another database, one whose xid is older +# (it holds every snapshot's horizon). Both compared through age() against one reference, +# which is exactly `< X`, wraparound-safe, and takes no xid. +_OPEN = """ +WITH x AS (SELECT age($1::text::xid8::xid) AS a) +SELECT s.pid, s.backend_type, s.application_name, s.xact_start::text AS xact_start, + s.datid = (SELECT oid FROM pg_database WHERE datname = current_database()) AS ours, + coalesce(s.backend_xid IS NOT NULL AND age(s.backend_xid) > x.a, false) AS xid_old, + coalesce(s.backend_xmin IS NOT NULL AND age(s.backend_xmin) > x.a + AND s.backend_type <> 'autovacuum worker', false) AS xmin_old, + coalesce(s.query = '', false) AS hidden +FROM pg_stat_activity s, x +WHERE s.pid <> pg_backend_pid() +""" + + +def _named(r: Any) -> str: + return ( + f"pid {r['pid']} ({r['backend_type']}, {r['application_name'] or 'no name'}, " + f"transaction since {r['xact_start']})" + ) + + +async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + """Wait until no backend of the cell's database holds an xid or a snapshot older than + `xmax`; Refused at once on a prepared transaction or an unreadable row, and past + `wait` naming each backend waited on and each other database's horizon holder.""" + while True: + if refusals := await prepared_refusals(conn): + raise Refused(refusals) + if await conn.fetchval("SELECT count(*) FROM pg_prepared_xacts"): + raise Refused( + [Refusal("postgres prepared transaction", "one is open: commit or roll it back")] + ) + rows = await conn.fetch(_OPEN, str(xmax)) + hidden = [r for r in rows if r["ours"] and r["hidden"]] + if hidden: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + "its transaction state is hidden from attach's role: grant it " + "pg_read_all_stats", + ) + for r in hidden + ] + ) + waited = [r for r in rows if r["ours"] and (r["xid_old"] or r["xmin_old"])] + if not waited: + return + if not await wait.pause(): + horizon = [r for r in rows if not r["ours"] and r["xid_old"]] + raise Refused( + [ + Refusal(f"postgres {_named(r)}", "may have read `recording`: still open") + for r in waited + ] + + [ + Refusal( + f"postgres {_named(r)}", + "another database's transaction, holding the snapshot horizon", + ) + for r in horizon + ] + ) + + +# -- the tenancy rows ------------------------------------------------------------------- + + +async def write_tenancy(orgs: Iterable[str]) -> int: + """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the + rows written.""" + batch = sorted(orgs) + written = 0 + for i in range(0, len(batch), 1000): + status = await pg.execute( + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) + written += int(status.split()[-1]) + return written + + +async def read_mode(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The cell's state (`cell.read_state`) and its `first` row.""" + state = await cell.read_state(conn) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + return state, None if first is None else (bool(first["is_first"]), str(first["cell"])) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 8ad83098a9..cf83fa8f27 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -1,7 +1,10 @@ """`loom cell prepare`: register this cell in the router's directory and write the rows its processes read (docs/architecture/cells.md, "Preparing a cell" and "The tenancy check"). -It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names): +It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names), +holding the attach lock (`cell.ATTACH_LOCK_KEY`) on a direct session of the cell's Postgres, +after that session's checks (one backend, the tenancy marker kept, a UTF8 database), so it +never runs beside an attach or another prepare: 1. It reads the cell's Postgres (deployment id, `first`, mode and own rows, tenancy rows) and the live processes' heartbeats, and refuses a cell whose rows name another cell @@ -23,11 +26,13 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets import sys import time +from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -46,6 +51,9 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound +# A bound on the lock session's statements: it reads the catalog and holds the lock. +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads + _sleep = asyncio.sleep # the seam the retry tests replace @@ -442,6 +450,38 @@ async def prepare( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, +) -> int: + """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" + async with attach_lock(): + return await _prepare_locked(ch, args, router, transport) + + +@contextlib.asynccontextmanager +async def attach_lock() -> AsyncIterator[None]: + """The attach lock, held on a direct session of the cell's Postgres while the block + runs, after that session's checks: prepare and attach write the same probe nonce, and + one of them runs at a time.""" + from server.cli import cell_attach_pg + from server.cli.cell_ch import Refused as Stopped + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + try: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + except Stopped as e: + raise Refused(str(e)) from None + yield + + +async def _prepare_locked( + ch: Any, + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None, ) -> int: rows = await _read_rows() if not rows.deployment_id: diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index 6968dd49b0..e7d9077a02 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -67,6 +67,19 @@ } ) +# Tables a worker drains into another table, deleting what it moved: each is scanned +# before every table that is not one, so a row moved mid-scan is read where it was or where +# it went. A table a materialized view reads is ordered the same way, from +# `system.tables` dependencies (`order_sources`). +MOVE_SOURCES = frozenset( + { + "loom.learnings_queue", + "loom.memory_code_link_queue", + "loom.memory_extraction_queue", + "loom.memory_webhook_outbox", + } +) + # A refreshable view's temporary table while a refresh exchanges its target. _EXCHANGE = re.compile(r"^\.tmp\.inner(_id)?\.") # A provider transition's copies (server/erasure/store/provider_copies.py). @@ -338,6 +351,32 @@ async def list_postgres(conn: Any) -> tuple[list[str], list[Refusal]]: return names, refusals +def order_sources(tables: list[Table], view_sources: Iterable[str]) -> list[Table]: + """`tables` with the move sources (`MOVE_SOURCES`, and every table a view reads, + `view_sources`, canonical names) first, each group by name.""" + first = MOVE_SOURCES | set(view_sources) + return sorted(tables, key=lambda t: (t.canonical not in first, t.canonical)) + + +async def view_sources( + ch: Any, topology: Topology, names: Mapping[str, str], dbs: list[str] +) -> set[str]: + """The tables of `dbs` a view depends on, as canonical names.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, database, name FROM {source:tables} " + "WHERE database IN {dbs:Array(String)} AND notEmpty(dependencies_table)", + {"dbs": dbs}, + ) + return { + f"{ch_names.logical(str(r['database']))}.{r['name']}" + for found in rows.values() + for r in found + } + + async def list_tables( ch: Any, pg_conn: Any, topology: Topology, names: Mapping[str, str] ) -> TableList: @@ -370,7 +409,8 @@ async def _list_tables( if copies := provider_copies(tables): out.refusals += copies return out - out.clickhouse, problems = judge_clickhouse(tables, classified, topology) + scanned, problems = judge_clickhouse(tables, classified, topology) + out.clickhouse = order_sources(scanned, await view_sources(ch, topology, names, databases())) out.refusals += problems out.refusals += await reverse_clickhouse(ch, tables, classified) out.postgres, problems = await list_postgres(pg_conn) @@ -460,7 +500,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def _pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -615,7 +655,7 @@ async def _read_scan( return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def _ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) @@ -697,10 +737,10 @@ async def scan( result.add("configured", org) with query_tag(TAG): for name in tables.postgres: - for org in await _pg_orgs(name, budget): + for org in await pg_orgs(name, budget): result.add(f"postgres {name}", org) for table in tables.clickhouse: - for raw in await _ch_orgs(ch, table, topology, budget): + for raw in await ch_orgs(ch, table, topology, budget): result.add(f"clickhouse {table.canonical}", raw) result.elapsed_s = budget.elapsed() return result diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..29a9ddabf6 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -566,10 +566,15 @@ def grade(p: Any) -> str: return [(f"problem:{p.code}", grade(p), f"{p.message} Fix: {p.fix}") for p in problems] -def _run_doctor_checks() -> list[tuple[str, str, str]]: +def _run_doctor_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The live checks (Python, settings, JWT, ClickHouse, LLM) followed - by the shared problem list, as (name, grade, detail).""" - return _run_live_checks() + _code_mirror_checks() + _problem_checks(_doctor_problems()) + by the shared problem list, as (name, grade, detail). `allowed_roles` is + `--pg-allow-role`, for a cell's checks.""" + return ( + _run_live_checks(allowed_roles) + + _code_mirror_checks() + + _problem_checks(_doctor_problems()) + ) def _code_mirror_checks() -> list[tuple[str, str, str]]: @@ -582,7 +587,7 @@ def _code_mirror_checks() -> list[tuple[str, str, str]]: return [("code_mirrors", WARN, f"not checked: {type(e).__name__}: {e}")] -def _run_live_checks() -> list[tuple[str, str, str]]: +def _run_live_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """Run the doctor's live checks and return them as (name, grade, detail) with grade one of PASS / WARN / FAIL. @@ -664,7 +669,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: if settings_ok and ch_ok and settings.cell: from server.cli import doctor_cell - checks += doctor_cell.cell_checks() + checks += doctor_cell.cell_checks(allowed_roles) # 5. LLM provider resolution: what extraction runs on and # what decided it; WARN when the provider was inferred (or named) but its @@ -677,6 +682,17 @@ def _run_live_checks() -> list[tuple[str, str, str]]: return checks +def _stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`--stray-orgs`, on a cell only (LOOM_CELL set).""" + from server.settings import settings + + if not settings.cell: + return [("cell_stray_orgs", WARN, "not read: LOOM_CELL is unset (no cell)")] + from server.cli import doctor_cell + + return doctor_cell.stray_org_checks(budget_s) + + def _client_checks(args: argparse.Namespace) -> list[tuple[str, str, str]]: """`loom doctor --client`: the checks from the machine a coding agent runs on, with the CLI's own HTTP helpers. A crash in one of @@ -722,7 +738,9 @@ def _cmd_doctor(_args: argparse.Namespace) -> int: operator sees the full picture. Exit 0 = nothing failed (warnings are listed but do not fail); 1 = at least one FAIL. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(tuple(getattr(_args, "pg_allow_role", None) or ())) + if getattr(_args, "stray_orgs", False): + checks += _stray_org_checks(_args.stray_orgs_budget) # The client's side of the same picture, when asked. client = _client_checks(_args) if getattr(_args, "client", False) else [] checks = _client_machine_grades(checks, client) + client @@ -841,4 +859,21 @@ def _add_doctor_flags(p_doc: argparse.ArgumentParser) -> None: ), ) p_doc.add_argument("--server-url", default=None, help="Override the client's server URL.") + p_doc.add_argument( + "--pg-allow-role", + action="append", + default=[], + help="On a cell: a Postgres role that may make an org held (the servers' role).", + ) + p_doc.add_argument( + "--stray-orgs", + action="store_true", + help="On a cell: also read the orgs its stores hold without a tenancy row.", + ) + p_doc.add_argument( + "--stray-orgs-budget", + type=float, + default=60.0, + help="Seconds --stray-orgs may read for, smallest tables first (default 60).", + ) p_doc.add_argument("--token", default=None, help="Override the client's token.") diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..c79723bbbc 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -1,7 +1,11 @@ """`loom doctor`'s cell checks, run only with LOOM_CELL set (docs/architecture/cells.md): the deployment check, a partial cell state, the tenancy marker on this process's -sessions and on every session, and the defects of the org triggers and the `loom.tenancy` -guard (`cell_triggers.missing`). +sessions and on every session, the defects of the org triggers and the `loom.tenancy` +guard (`cell_triggers.missing`), and the point-in-time reads attach makes, run again on +every doctor run: the login roles that can make an org held or lower the mode, and the +role and database settings and subscriptions that pass an unchecked writer +(`cell_attach_pg`). `--stray-orgs` also reads the orgs the cell's stores hold without a +tenancy row (`stray_org_checks`). A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; @@ -78,14 +82,34 @@ async def _unmarked_session() -> bool: """Whether a session that sends no marker passes the triggers all the same (a role, database, ALTER SYSTEM or options setting): marked, or in the replica role that fires no origin trigger.""" + from server.cli import cell_attach_pg + + return await cell_attach_pg.session_passes() + + +async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, str]]: + """Attach's point-in-time Postgres reads, again: the login roles outside the owner + roles, the superusers and `--pg-allow-role` that can make an org held or lower the + mode, and the settings and subscriptions that pass an unchecked writer.""" + from server.cli import cell_attach_pg from server.core import pg - async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS, marked=False) as conn: - value: str | None = await conn.fetchval( - "SELECT current_setting($1, true)", pg.TENANCY_MARKER - ) - role: str = await conn.fetchval("SELECT current_setting('session_replication_role')") - return value == "on" or role == "replica" + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + holders = await cell_attach_pg.privilege_holders(conn) + unchecked = await cell_attach_pg.unmarked_refusals(conn, False) + roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) + return [ + ( + "cell_tenancy_privilege", + WARN if roles else PASS, + "; ".join(str(r) for r in roles) or "no role outside the owner roles can", + ), + ( + "cell_unchecked_writers", + WARN if unchecked else PASS, + "; ".join(str(r) for r in unchecked) or "no setting or subscription passes one", + ), + ] def _grades( @@ -149,7 +173,7 @@ def _grades( return out -async def _checks(ch: Any) -> list[tuple[str, str, str]]: +async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): return [("cell_state", PASS, f"cell checks not run: {before}")] @@ -159,7 +183,11 @@ async def _checks(ch: Any) -> list[tuple[str, str, str]]: return [("cell_deployment", FAIL, f"deployment check refused: {e}")] except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + try: + reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) + except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run + reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + return _grades(state, marked, marks_all, defects) + reads def _reason(e: BaseException) -> str: @@ -167,9 +195,10 @@ def _reason(e: BaseException) -> str: return f"{type(e).__name__}: {str(e) or 'no answer within the bound'}" -def cell_checks() -> list[tuple[str, str, str]]: +def cell_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The cell checks, on the CLI's ClickHouse client connected for them alone (no CLI - deployment check: a deployment not booted yet is named, not refused).""" + deployment check: a deployment not booted yet is named, not refused). `allowed_roles` + is doctor's `--pg-allow-role`.""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client @@ -180,10 +209,121 @@ async def _go() -> list[tuple[str, str, str]]: connect_declared(ch, org_free=reads("loom doctor reads a cell's state")), timeout=_SESSION_S, ) - return await _checks(ch) + return await _checks(ch, allowed_roles) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: await disconnect(ch) return asyncio.run(_go()) + + +# -- --stray-orgs -------------------------------------------------------------------------- + + +async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: + """Each ClickHouse table's active row count, from `system.parts` (KeeperMap: 0).""" + from server.core import ch_names + + rows = await ch.query_all( + "SELECT database, table, sum(rows) AS n FROM system.parts WHERE active " + "AND database IN {dbs:Array(String)} GROUP BY database, table", + {"dbs": sorted({t.database for t in tables})}, + ) + return {f"{ch_names.logical(str(r['database']))}.{r['table']}": int(r["n"]) for r in rows} + + +async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: + from server.cli import cell_scan + from server.cli.cell_ch import Refused, databases, read_topology + from server.core import pg + from server.core.cell_triggers import ORG_TABLES + + topology = await read_topology(ch) + if topology.multi: + from server.cli.cell_writers import host_names + + names = await host_names(ch, topology) + else: + names = {} + listed, problems = await cell_scan.list_clickhouse(ch, topology, names, databases()) + ch_tables = [t for t in listed if t.engine.endswith("MergeTree") or t.engine == "KeeperMap"] + sizes = await _table_sizes(ch, ch_tables) + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + held = {str(r["org"]) for r in await conn.fetch("SELECT org FROM loom.tenancy")} + pg_rows = await conn.fetch( + "SELECT 'loom.' || c.relname AS name, greatest(c.reltuples, 0)::bigint AS n " + "FROM pg_class c WHERE c.oid IN (SELECT to_regclass('loom.' || t.relname) FROM (" + + ORG_TABLES + + ") t)" + ) + # Smallest first: a budget that runs out leaves the largest unchecked. + work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] + work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] + work.sort(key=lambda w: (w[0], str(w[2]))) + budget = cell_scan.Budget(budget_s) + stray: dict[str, list[str]] = {} + unchecked: list[str] = [] + for i, (_n, store, table) in enumerate(work): + found = cell_scan.ScanResult() + name = table if store == "postgres" else table.canonical + try: + if store == "postgres": + for org in await cell_scan.pg_orgs(table, budget): + found.add(name, org) + else: + for raw in await cell_scan.ch_orgs(ch, table, topology, budget): + found.add(name, raw) + except Refused: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) + bad = [m for ms in found.malformed.values() for m in ms] + if orgs or bad: + log = " (a log table: never placed)" if name in cell_scan.LOG_TABLES else "" + stray[f"{store} {name}{log}"] = orgs[:5] + bad[:5] + out = [ + ( + "cell_stray_orgs", + WARN if stray else PASS, + "; ".join(f"{t}: {', '.join(o)}" for t, o in sorted(stray.items())) + or "every org found has a tenancy row", + ) + ] + if unchecked or problems: + out.append( + ( + "cell_stray_orgs_unchecked", + WARN, + f"not read within --stray-orgs-budget ({budget_s:g} s): " + + ", ".join(unchecked + [str(p) for p in problems]), + ) + ) + return out + + +def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row + (log-only orgs, names no request can carry, rows an unchecked writer left), smallest + tables first within one budget, naming every table left unchecked. Never a FAIL: the + stray-org erase command removes what it names.""" + from server.cli._ch import connect_declared, disconnect + from server.core.ch import get_client, statement_timeout + + async def _go() -> list[tuple[str, str, str]]: + ch = get_client() + try: + await asyncio.wait_for( + connect_declared(ch, org_free=reads("loom doctor reads a cell's orgs")), + timeout=_SESSION_S, + ) + if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): + return [("cell_stray_orgs", PASS, f"not read: {before}")] + return await _stray_reads(ch, budget_s) + except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run + return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] + finally: + await disconnect(ch) + + with statement_timeout(budget_s): + return asyncio.run(_go()) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 19cf492b0e..9c2c9755ae 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -36,7 +36,9 @@ CELL_FIRST_TRIGGERS = ("cell_first_immutable", "cell_first_no_truncate") # Org-keyed tables the org triggers skip: the cell's own tenancy rows (`tenancy_record` # guards them instead), and the idempotency rows keyed by the caller's token org (a platform -# admin's replay row for its home org is neither refused nor placed). +# admin's replay row for its home org is not refused). Outside the guard is not outside the +# first attach: its org scan (`server/cli/cell_scan.py`) reads every table with an `org` +# column, these included, and places the orgs it finds. EXCLUDED = frozenset({"tenancy", "idempotency", "idempotency_fences"}) # A table's DDL waits this long for its lock a try, LOCK_ATTEMPTS tries LOCK_RETRY_S # apart: a boot never queues behind a long transaction (and every write of the table @@ -77,10 +79,13 @@ # What a trigger's DDL (SHARE ROW EXCLUSIVE) waited behind on a table, read after the # wait failed (pg_blocking_pids is empty by then): the locks held in a mode it conflicts # with (every mode but the two readers take), and the requests queued ahead of it in -# one, whose own blocker heads the queue. +# one, whose own blocker heads the queue. An anti-wraparound autovacuum, which never yields +# its lock, says so in its activity line (compared here, never returned). _HOLDERS = """ SELECT DISTINCT l.pid, l.granted, a.backend_type, a.state, a.application_name, - a.xact_start, a.wait_event + a.xact_start, a.wait_event, + coalesce(a.backend_type = 'autovacuum worker' + AND a.query LIKE '%(to prevent wraparound)', false) AS wraparound FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = $1::regclass AND l.pid <> pg_backend_pid() AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) @@ -208,63 +213,107 @@ async def ensure(conn: Any) -> None: tries; a table whose lock it cannot take is logged with the holders of its locks, one the role cannot alter is logged as such, and either is skipped (the boot serves). A failed read of the state or the catalog raises. Runs on the migration session, which - holds the migration lock.""" + holds the migration lock. `ensure_table` runs one table's part alone.""" state = await cell.read_state(conn) if state == "open": return - # A table's statements, in one transaction: one lock wait, and every statement or none. - statements: dict[str, list[str]] = {} first = await _triggers(conn, CELL_FIRST_TRIGGERS) - for name in CELL_FIRST_TRIGGERS: - row = first.get(("cell_first", name)) - if row is not None and row["enabled"] != "A": - statements.setdefault("cell_first", []).append( - f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + always = [ + f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + for name in CELL_FIRST_TRIGGERS + if (row := first.get(("cell_first", name))) is not None and row["enabled"] != "A" + ] + if always: + await _run_ddl(conn, "cell_first", always) + if state != "enforced": + return + tables = await conn.fetch(ORG_TABLES) + for r in tables: + if r["relkind"] == "p": + # No trigger guards it: named here as missing() names it. + log.warning( + "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] ) - if state == "enforced": - present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) - tables = await conn.fetch(ORG_TABLES) - for r in tables: - if r["relkind"] == "p": - # No trigger guards it: named here as missing() names it. - log.warning( - "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] - ) - for r, names in _planned(tables): - table = r["relname"] - for name in names: - row = present.get((table, name)) - if row is None: - statements.setdefault(table, []).append(_create(f"loom.{_ident(table)}", name)) - elif row["enabled"] != "O": - statements.setdefault(table, []).append( - f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}" - ) - for table, sqls in statements.items(): - await _run_ddl(conn, table, sqls) + present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) + for r, names in _planned(tables): + if sqls := _table_statements(r["relname"], names, present): + await _run_ddl(conn, r["relname"], sqls) + + +async def planned_tables(conn: Any) -> list[str]: + """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" + return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + + +async def ensure_table( + conn: Any, + table: str, + *, + recording_first: bool = False, + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its + org triggers), so one table's DDL can run alone: in `enforced`, or with + `recording_first` also in a first cell's `recording`, where the org triggers' function + passes every row and a boot never creates them. False when the DDL was not applied (its + lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); + True when it was, or nothing was needed. Raises for a table outside the planned set, or + in another state.""" + state = await cell.read_state(conn) + if state != "enforced" and not (recording_first and state == "recording"): + raise RuntimeError(f"cell triggers: not created in state {state!r}") + planned = [ + (r, names) for r, names in _planned(await conn.fetch(ORG_TABLES)) if r["relname"] == table + ] + if not planned: + raise ValueError(f"loom.{table} carries no cell triggers") + ((_, names),) = planned + sqls = _table_statements(table, names, await _triggers(conn, names)) + return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + + +def _table_statements( + table: str, names: tuple[str, ...], present: dict[tuple[str, str], Any] +) -> list[str]: + """The DDL that gives `table` its named triggers, each present and enabled.""" + out = [] + for name in names: + row = present.get((table, name)) + if row is None: + out.append(_create(f"loom.{_ident(table)}", name)) + elif row["enabled"] != "O": + out.append(f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}") + return out -async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries. The boot serves past - the two causes cells.md names (its lock not taken, a table this role cannot alter); - any other error fails it, as a failed read of the state does.""" +async def _run_ddl( + conn: Any, + table: str, + sqls: list[str], + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The + boot serves past the two causes cells.md names (its lock not taken, a table this role + cannot alter); any other error fails it, as a failed read of the state does.""" import asyncpg - for attempt in range(LOCK_ATTEMPTS): + for attempt in range(attempts): try: async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return + return True except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return + return False except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. - if attempt < LOCK_ATTEMPTS - 1: + if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue log.warning( @@ -274,13 +323,19 @@ async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: "; ".join(sqls), await _holders(conn, table), ) + return False + + +async def holders(conn: Any, table: str) -> list[Any]: + """The sessions a blocked DDL on `table` waits behind (`_HOLDERS`).""" + return list(await conn.fetch(_HOLDERS, f"loom.{_ident(table)}")) async def _holders(conn: Any, table: str) -> str: """The sessions a blocked DDL waited behind, never their query text (a literal in it may be memory content).""" try: - rows = await conn.fetch(_HOLDERS, f"loom.{_ident(table)}") + rows = await holders(conn, table) except Exception as e: # noqa: BLE001 — naming the holders is best effort log.warning("cell triggers: the lock holders of loom.%s not read: %s", table, e) return "lock holders not read" diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..bb47bfd945 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -53,6 +53,10 @@ _LIVE_INTERVALS = 3 # How often the leader drops the KeeperMap keys of processes gone an hour. _PRUNE_EVERY_S = 600.0 +# How old a row's last beat is before the leader drops its key. A contract across builds, +# not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader +# that sweeps them may run another build, so a change is a design change of its own. +PRUNE_AFTER_S = 3600 # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -130,6 +134,22 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + +def boot_allowance_s() -> float: + """How long a booting process takes to report its first cell row and its first router + answer: two beats and one router read.""" + return 2 * float(settings.instance_heartbeat_seconds) + float(settings.cell_router_timeout_s) + + async def beat(ch: Any, *, stopped: bool = False) -> None: """Write this process's row. A KeeperMap insert replaces the key.""" await ch.insert(TABLE, [row(stopped=stopped)]) @@ -197,6 +217,34 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out +async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose widened window + (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or + past now without one: stopped rows included, which `read_live` drops. `stopped` is its + newest row's flag and `live` whether that row is inside the live window and not + stopped, both judged in ClickHouse.""" + rows = await ch.query_all( + f""" + SELECT instance_id, + argMax(keys_fp, heartbeat_at) AS keys_fp, + argMax(stopped, heartbeat_at) AS stopped, + toString(max(heartbeat_at)) AS last_beat, + max(heartbeat_at) > now64(3) - toIntervalMillisecond({{live:UInt32}}) + AND stopped = 0 AS live + FROM {TABLE} + GROUP BY instance_id + HAVING max(heartbeat_at) + toIntervalMillisecond({{wide:UInt32}}) + > if({{since:String}} = '', now64(3), toDateTime64({{since:String}}, 3)) + """, + params={ + "live": int(live_window_s() * 1000), + "wide": int(widened_window_s() * 1000), + "since": since or "", + }, + ) + return [dict(r) for r in rows] + + def live_rows() -> list[dict[str, Any]]: """The live rows as of the last read; none before the first read or once that read is older than the live window (ClickHouse unreachable).""" @@ -224,14 +272,15 @@ async def _prune(ch: Any) -> None: # Each table's sweep stands alone: one failing leaves the other to run. try: await ch.execute( - "ALTER TABLE loom.instances DELETE WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + "ALTER TABLE loom.instances DELETE " + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("instance row sweep failed: %s", e) try: await ch.execute( "ALTER TABLE loom.cell_heartbeats DELETE " - "WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("cell heartbeat sweep failed: %s", e) @@ -331,8 +380,10 @@ def _reset_for_tests() -> None: __all__ = [ "CELL_TABLE", + "PRUNE_AFTER_S", "TABLE", "beat", + "boot_allowance_s", "cell_beat", "cell_row", "instance_id", @@ -341,8 +392,10 @@ def _reset_for_tests() -> None: "live_window_s", "publish_config", "read_live", + "read_recent", "schema_version", "start", "stop", "tick", + "widened_window_s", ] diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py new file mode 100644 index 0000000000..406d277646 --- /dev/null +++ b/tests/integration/test_cell_attach.py @@ -0,0 +1,635 @@ +"""`loom cell attach` (server/cli/cell_attach.py) against a real Postgres with a synchronous +standby, a real ClickHouse and the router served in process: a prepared first cell in +`recording` with orgs in both stores attaches to `serving` with every org placed and the +triggers created; a rerun on `serving` exits 0; a refusal after step 3 leaves `enforced` and +a rerun resumes from there; a second run is refused at the attach lock; the Postgres reads +attach makes refuse what they name (an unchecked writer's setting, a subscription, a role +that can make an org held, a transaction that may have read `recording`). + +The cell's Postgres and the directory are databases of the test's own; the configured +ClickHouse databases are scratch ones; the cell's processes are heartbeat rows the test +writes. Needs LOOM_PG_DSN (a server with a synchronous standby), LOOM_ROUTER_PG_DSN and +LOOM_CH_HOST.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell as cell_cli +from server.cli import cell_attach, cell_attach_pg, cell_prepare +from server.cli.cell_ch import Refused +from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core.ch import get_client +from server.core.schema import pg_migrate +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +OPERATOR = "lco_" + "2" * 128 +CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() +ROUTER = "http://127.0.0.1:7901" +HOST = "loom.example" + + +class _Network(httpx.AsyncBaseTransport): + """What attach and the router reach: the router at ROUTER, the shared host's health + route, the cell's upstream main port (:7777) and its probe listener (any other).""" + + def __init__(self) -> None: + self.listener = httpx.ASGITransport(app=cell_probe.listener_app()) + self.router = httpx.ASGITransport( + app=router_app.build_app(router_app.Router(probe_transport=self)) + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "127.0.0.1" and request.url.port == 7901: + return await self.router.handle_async_request(request) + if request.url.host == HOST or request.url.port == 7777: + return httpx.Response(200, json=cell_probe.answer()) + return await self.listener.handle_async_request(request) + + +async def _admin(sql: str, *args: Any) -> Any: + import asyncpg + + conn = await asyncpg.connect(os.environ["LOOM_PG_DSN"]) + try: + return await conn.execute(sql, *args) + finally: + await conn.close() + + +@pytest.fixture +async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: + cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") + cell_db, own_cell = await create_database(cell_dsn) + router_db, own_router = await create_database(router_dsn) + deployment = f"dep-{uuid.uuid4().hex[:8]}" + tag = f"it-{uuid.uuid4().hex[:8]}-" + for name, value in { + "cell": "first", + "storage_authority": "postgres", + "pg_dsn": own_cell, + "document_s3_bucket": "", + "router_pg_dsn": own_router, + "router_shared_host": HOST, + "router_upstream_probe_url": "http://listener.internal:7790", + "router_upstream_url": "http://upstream.internal:7777", + "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), + "router_operator_token": OPERATOR, + "ch_cluster": "", + "public_base_url": f"https://{HOST}", + "oauth_issuer": "", + }.items(): + monkeypatch.setattr(settings, name, value) + await pg.close_pool() + monkeypatch.setattr(directory, "_pool", None) + # A step's poll interval and the waits stay short. + monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) + ch = get_client() + await ch.connect() + ids: list[str] = [] + try: + await pg_migrate.migrate() + await pg.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) " + "VALUES ('postgres', 1, $1)", + deployment, + ) + await directory.migrate() + await ch.insert( + "loom.storage_authority", + [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + ) + cell.set_deployment_id(deployment) + yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} + finally: + cell.set_deployment_id("") + await ch.execute( + "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " + "SETTINGS mutations_sync = 2", + {"d": deployment}, + ) + for table in (instances.TABLE, instances.CELL_TABLE): + await ch.execute( + f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", + {"ids": ids}, + ) + await ch.execute( + "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} + ) + await ch.disconnect() + await pg.close_pool() + await directory.close() + await drop_database(cell_dsn, cell_db) + await drop_database(router_dsn, router_db) + + +def _value(**keys: Any) -> str: + reported = { + **cell.as_value(cell.mine()), + "serves": True, + "marked": True, + "router": True, + "router_why": "", + **keys, + } + return json.dumps(reported, sort_keys=True) + + +async def _beat( + stores: dict[str, Any], + instance_id: str, + *, + cell_name: str | None = "first", + mode: str = "recording", + stopped: bool = False, + **keys: Any, +) -> None: + """One process's heartbeat rows, as `instances.tick` writes them.""" + ch = stores["ch"] + if instance_id not in stores["ids"]: + stores["ids"].append(instance_id) + await ch.insert( + instances.TABLE, [{**instances.row(stopped=stopped), "instance_id": instance_id}] + ) + if cell_name is not None: + await ch.insert( + instances.CELL_TABLE, + [ + { + "instance_id": instance_id, + "cell": cell_name, + "tenancy_mode": mode, + "fingerprint": _value(**keys), + } + ], + ) + + +async def _prepare(stores: dict[str, Any], network: _Network) -> None: + pid = f"it-{uuid.uuid4().hex[:8]}" + await _beat(stores, pid, cell_name="first", mode="open") + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + assert await cell_prepare.prepare(stores["ch"], args, router, network) == 0 + stores["pid"] = pid + await _beat(stores, pid, mode="recording") + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 20, + "scan_budget": 600, + "dry_run": False, + "ch_allow_user": [settings.ch_user or "default"], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": ROUTER, + } + return argparse.Namespace(**{**fields, **over}) + + +async def _attach(stores: dict[str, Any], network: _Network, **over: Any) -> int: + args = _args(**over) + url = cell_cli.operator_router(args) + assert isinstance(url, httpx.URL) + return await cell_attach.attach(stores["ch"], args, url, network) + + +async def _mode() -> str: + return str(await pg.fetchval("SELECT mode FROM loom.tenancy_mode")) + + +async def _placed() -> set[str]: + import asyncpg + + conn = await asyncpg.connect(settings.router_pg_dsn) + try: + rows = await conn.fetch("SELECT org FROM loom_router.placements WHERE cell = 'first'") + finally: + await conn.close() + return {r["org"] for r in rows} + + +async def _row() -> Any: + return await directory.first_cell_named("first") + + +@pytest.mark.asyncio +async def test_a_recording_first_cell_attaches_to_serving(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + tag = stores["tag"] + await _prepare(stores, network) + capsys.readouterr() + # Orgs a recording cell holds: a tenancy row alone, a Postgres data row, a ClickHouse + # row, and one name no request can carry (skipped and named). + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "signed-in") + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "pg") + await stores["ch"].insert( + "loom.namespaces", + [ + {"org": tag + "ch", "namespace": "n", "owner_email": ""}, + {"org": tag + "bad\x00org", "namespace": "n", "owner_email": ""}, + ], + ) + assert await _attach(stores, network) == 0 + report = json.loads(capsys.readouterr().out) + mine = {tag + "signed-in", tag + "pg", tag + "ch"} + assert report["state"] == "serving" + assert await _mode() == "enforced" + placed = await _placed() + assert {o for o in placed if o.startswith(tag)} == mine + held = {r["org"] for r in await pg.fetch("SELECT org FROM loom.tenancy")} + assert held == placed + assert any("clickhouse loom.namespaces" in k for k in report["skipped"]["malformed"]) + row = await _row() + assert (row.state, row.mode, row.placed) == ("serving", "enforced", len(placed)) + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + # A rerun on `serving` is done. + assert await _attach(stores, network) == 0 + assert json.loads(capsys.readouterr().out)["done"] is True + + +@pytest.mark.asyncio +async def test_a_refusal_after_step_3_leaves_enforced_and_a_rerun_resumes( + cell_stores, monkeypatch, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "a" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + # A process with LOOM_CELL unset appears once the mode is committed: refused at once. + real = cell_attach.Run.durable + + async def durable_then_unset(self: Any) -> int: + xmax = await real(self) + await _beat(stores, f"it-unset-{uuid.uuid4().hex[:6]}", cell_name="") + return xmax + + monkeypatch.setattr(cell_attach.Run, "durable", durable_then_unset) + assert await _attach(stores, network) == 1 + assert "runs with LOOM_CELL unset" in capsys.readouterr().err + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("attaching", "enforced") + # Placed live once enforced; the rerun resumes from the durable read. + monkeypatch.setattr(cell_attach.Run, "durable", real) + for i in list(stores["ids"]): + if i.startswith("it-unset-"): + await _beat(stores, i, cell_name="first", mode="enforced") + await _beat(stores, stores["pid"], mode="enforced") + assert await _attach(stores, network) == 0 + assert org in await _placed() + assert (await _row()).state == "serving" + + +@pytest.mark.asyncio +async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "the test") + assert await _attach(stores, network) == 1 + assert "attach lock: held by pid" in capsys.readouterr().err + assert await _mode() == "recording" + + +@pytest.mark.asyncio +async def test_a_dry_run_writes_nothing(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + capsys.readouterr() + org = stores["tag"] + "dry" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + assert await _attach(stores, network, dry_run=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report["per_source"]["postgres loom.scope_generations"] == { + "count": 1, + "sample": [org], + } + assert await _mode() == "recording" + assert await _placed() == set() + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with pg.dedicated() as conn: + assert any(d.kind == "absent" for d in await cell_triggers.missing(conn)) + + +@pytest.mark.asyncio +async def test_an_org_past_the_limit_refuses_before_any_tenancy_row(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + long = "o" * (cell.ORG_MAX_BYTES + 1) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", long) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('org-ok', '*')") + assert await _attach(stores, network) == 1 + assert f"longer than {cell.ORG_MAX_BYTES} bytes" in capsys.readouterr().err + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert (await _row()).placed == 0 + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_that_pass_unchecked_writers_refuse( + cell_stores, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + db = stores["db"] + role = f"it_role_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + # Stored as first spelled: a fresh session that never set the placeholder. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" SET "Loom.Tenancy_Checked" = \'on\'') + async with pg.dedicated() as conn: + setting = await conn.fetchval( + "SELECT setconfig::text FROM pg_db_role_setting s JOIN pg_roles r " + "ON r.oid = s.setrole WHERE r.rolname = $1", + role, + ) + assert "Loom.Tenancy_Checked" in setting + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [r.subject for r in refusals] == [f"postgres setting for {role} in {db}"] + # Another database's entry does not refuse. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" RESET ALL') + await _admin( + f"ALTER ROLE {role} IN DATABASE postgres SET session_replication_role = 'replica'" + ) + async with pg.dedicated() as conn: + assert await cell_attach_pg.unmarked_refusals(conn, False) == [] + # A role that can insert a tenancy row, a column grant alone, refuses attach. + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + assert await _attach(stores, network) == 1 + err = capsys.readouterr().err + assert f"postgres role {role}: may insert into loom.tenancy" in err + assert await _mode() == "recording" + assert await _attach(stores, network, pg_allow_role=[role], dry_run=True) == 0 + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} IN DATABASE postgres RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_disabled_subscription_refuses(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + name = f"it_sub_{uuid.uuid4().hex[:6]}" + await pg.execute( + f"CREATE SUBSCRIPTION {name} CONNECTION 'host=127.0.0.1 dbname=none' " + "PUBLICATION none WITH (connect = false, enabled = false)" + ) + try: + async with pg.dedicated() as conn: + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [str(r) for r in refusals] == [ + f"postgres subscription {name}: disabled: its apply worker fires no origin " + "trigger; drop it" + ] + finally: + await pg.execute(f"ALTER SUBSCRIPTION {name} SET (slot_name = NONE)") + await pg.execute(f"DROP SUBSCRIPTION {name}") + + +@pytest.mark.asyncio +async def test_step_4_waits_for_a_transaction_that_may_have_read_recording(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as writer, pg.dedicated() as poller: + tx = writer.transaction() + await tx.start() + await writer.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('w', '*')") + # X is read after a later transaction completed, as after the durable read. + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + # A session idle outside a transaction holds nothing: the wait passes. + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + + +@pytest.mark.asyncio +async def test_the_session_checks_and_the_encoding(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as conn: + assert await cell_attach_pg.direct_refusals(conn, "attach") == [] + assert await cell_attach_pg.encoding_refusals(conn) == [] + async with pg.dedicated(marked=False) as conn: + refusals = await cell_attach_pg.direct_refusals(conn, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + f"attach's session loses {pg.TENANCY_MARKER}" + ] + name = f"it_latin_{uuid.uuid4().hex[:6]}" + await _admin( + f"CREATE DATABASE {name} ENCODING 'LATIN1' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0" + ) + try: + import asyncpg + + from tests.integration._pg_database import with_database + + conn = await asyncpg.connect(with_database(os.environ["LOOM_PG_DSN"], name)) + try: + (refusal,) = await cell_attach_pg.encoding_refusals(conn) + assert "LATIN1, not UTF8" in refusal.reason + finally: + await conn.close() + finally: + await _admin(f"DROP DATABASE {name}") + + +@pytest.mark.asyncio +async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + logged: list[str] = [] + + async def ok() -> bool: + return True + + async with pg.dedicated() as holder, pg.dedicated() as conn: + tx = holder.transaction() + await tx.start() + await holder.execute("SELECT mode FROM loom.tenancy_mode FOR SHARE") + + async def release() -> None: + await asyncio.sleep(3) + await tx.rollback() + + task = asyncio.create_task(release()) + await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await task + assert await _mode() == "enforced" + assert logged == ["the mode's lock is busy; retrying"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "grant,why", + [ + ("GRANT INSERT (org) ON loom.tenancy TO {r}", "may insert into loom.tenancy"), + ("GRANT UPDATE (org) ON loom.tenancy TO {r}", "may update loom.tenancy (org)"), + ("GRANT UPDATE (mode) ON loom.tenancy_mode TO {r}", "may update loom.tenancy_mode (mode)"), + ("GRANT INSERT (mode) ON loom.tenancy_mode TO {r}", "may insert into loom.tenancy_mode"), + ( + "GRANT INSERT, TRUNCATE ON loom.tenancy_mode TO {r}", + "may insert into loom.tenancy_mode", + ), + ( + "GRANT TRIGGER ON loom.scope_generations TO {r}", + "holds TRIGGER on loom.scope_generations", + ), + ("GRANT CREATE ON SCHEMA public TO {r}", "may create in schema public"), + ("GRANT CREATE ON SCHEMA loom TO {r}", "may create in schema loom"), + ( + "GRANT SET ON PARAMETER session_replication_role TO {r}", + "may set session_replication_role", + ), + ( + "SET loom.tenancy_checked = 'off'; " + "GRANT ALTER SYSTEM ON PARAMETER loom.tenancy_checked TO {r}", + "may ALTER SYSTEM SET loom.tenancy_checked", + ), + ("GRANT pg_write_server_files TO {r}", "is a member of pg_write_server_files"), + # A member of the owner role, ADMIN only: it can grant itself the rest. + ("GRANT {owner} TO {r} WITH ADMIN TRUE, INHERIT FALSE, SET FALSE", "owns loom.tenancy"), + ], +) +async def test_each_right_that_makes_an_org_held_is_named( + cell_stores, grant: str, why: str +) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_r_{uuid.uuid4().hex[:6]}" + owner = await pg.fetchval("SELECT current_user") + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) + for sql in grant.format(r=role, owner=owner).split("; "): + await conn.execute(sql) + holders = await cell_attach_pg.privilege_holders(conn) + assert owner in holders.own + assert any(why in w for w in holders.others.get(role, [])), holders.others + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_f_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") + holders = await cell_attach_pg.privilege_holders(conn) + await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") + assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] + finally: + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_repeatable_read_snapshot_from_before_x_holds_step_4(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as reader, pg.dedicated() as poller: + tx = reader.transaction(isolation="repeatable_read") + await tx.start() + await reader.fetchval("SELECT mode FROM loom.tenancy_mode") # its snapshot, no xid + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + + +@pytest.mark.asyncio +async def test_prepare_is_refused_while_attach_holds_the_lock(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "attach") + with pytest.raises(cell_prepare.Refused, match="attach lock: held by pid"): + await cell_prepare.prepare(cell_stores["ch"], args, router, _Network()) + + +@pytest.mark.asyncio +async def test_doctor_names_the_same_holders_settings_and_subscriptions(cell_stores) -> None: + from server.cli import doctor_cell + + await _prepare(cell_stores, _Network()) + role = f"it_d_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await _admin(f"ALTER ROLE {role} SET session_replication_role = 'replica'") + lines = {n: (g, d) for n, g, d in await doctor_cell._writer_reads(())} + assert lines["cell_tenancy_privilege"][0] == "WARN" + assert f"postgres role {role}" in lines["cell_tenancy_privilege"][1] + assert lines["cell_unchecked_writers"][0] == "WARN" + allowed = {n: g for n, g, _d in await doctor_cell._writer_reads((role,))} + assert allowed["cell_tenancy_privilege"] == "PASS" + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "held") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "held" + ) + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + grade, detail = lines["cell_stray_orgs"] + assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail + tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} + assert tight["cell_stray_orgs_unchecked"][0] == "WARN" diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index d1892be846..c78bc40331 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -166,7 +166,7 @@ def walk(node: dict[str, Any]) -> None: ("Index Only Scan", "tenancy_pkey"), ("Index Only Scan", "tenancy_pkey"), ], scans - assert len(await cell_scan._pg_orgs("loom.tenancy", Budget(60))) == 20000 + assert len(await cell_scan.pg_orgs("loom.tenancy", Budget(60))) == 20000 # Every org-keyed table of the schema has the index the scan needs. names, refusals = await cell_scan.list_postgres(own_pg) assert refusals == [] diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py new file mode 100644 index 0000000000..015cebd948 --- /dev/null +++ b/tests/unit/test_cell_attach.py @@ -0,0 +1,682 @@ +"""`loom cell attach` (server/cli/cell_attach.py, cell_attach_beats.py, cell_attach_pg.py) on +fakes: every refusal the heartbeats, the Postgres reads and the router's answers make, the +waits and their budget, the placement batches and the rescan's watch. The real Postgres, +ClickHouse and router behaviour is tests/integration/test_cell_attach.py. CH-free.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest + +from server.cli import cell_attach, cell_scan +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal, Refused +from server.cli.cell_scan import Table +from server.core import cell, instances +from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES +from server.settings import settings + +FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} +CURRENT = cell.as_value(FIELDS) +NEXT = cell.as_value({**FIELDS, "oidc_client_id": "next"}) + + +def _value(**keys: Any) -> dict[str, Any]: + return {**CURRENT, "serves": True, "marked": True, "router": True, "router_why": "", **keys} + + +def _beat( + i: str = "p1", + *, + cell_name: str | None = "first", + live: bool = True, + stopped: bool = False, + mode: str = "recording", + value: dict[str, Any] | None = None, + keys_fp: str = "k", +) -> beats.Beat: + return beats.Beat( + instance_id=i, + live=live, + stopped=stopped, + keys_fp=keys_fp, + last_beat="2026-01-01 00:00:00.000", + cell=cell_name, + mode=mode if cell_name is not None else "", + value=(value if value is not None else _value()) if cell_name is not None else None, + ) + + +def _judge(found: list[beats.Beat], stopped_unset: beats.StoppedUnset = "refuse") -> Any: + return beats.judge(found, "first", (CURRENT, NEXT), stopped_unset) + + +def _reasons(verdict: beats.Verdict) -> list[str]: + return [str(r) for r in verdict.refusals] + + +# -- the heartbeats -------------------------------------------------------------------- + + +def test_a_healthy_cell_passes_and_reports_its_keys() -> None: + nxt = {**_value(), **NEXT} + verdict = _judge([_beat("a"), _beat("b", value=nxt)]) + assert (verdict.refusals, verdict.booting, verdict.live, verdict.keys_fps) == ([], [], 2, {"k"}) + + +def test_a_live_process_with_loom_cell_unset_or_another_cell_refuses_at_once() -> None: + verdict = _judge([_beat("a", cell_name=""), _beat("b", cell_name="other")], "wait") + assert _reasons(verdict) == [ + "process a: runs with LOOM_CELL unset: set LOOM_CELL=first on it", + "process b: runs with LOOM_CELL=other: set LOOM_CELL=first on it", + ] + + +def test_a_stopped_unset_process_refuses_or_is_waited_on_as_asked() -> None: + stopped = [ + _beat("a", cell_name="", live=False, stopped=True), + _beat("b", cell_name=None, live=False, stopped=True), + ] + refused = _judge(stopped, "refuse") + assert [r.subject for r in refused.refusals] == ["process a", "process b"] + assert "may still be writing" in refused.refusals[0].reason + waited = _judge(stopped, "wait") + assert (waited.refusals, waited.aging) == ([], ["a", "b"]) + + +def test_a_live_instance_with_no_cell_row_is_booting() -> None: + verdict = _judge([_beat("a", cell_name=None)]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_cell_row_without_the_current_keys_refuses_even_while_booting() -> None: + old = {k: v for k, v in _value().items() if k not in ("serves", "marked", "router")} + verdict = _judge([_beat("a", value=old, mode="")]) + assert _reasons(verdict) == [ + "process a: reports no serves, marked or router key: upgrade it to this release" + ] + + +@pytest.mark.parametrize( + "keys,booting,refused", + [ + ({"marked": False}, [], "lose the tenancy marker"), + ({"marked": None}, ["a"], ""), + ({"router": False, "router_why": "a 404"}, [], "cannot read its router: a 404"), + ({"router": None}, ["a"], ""), + # A workers-only pod places no org: it needs no router credential. + ({"serves": False, "router": False}, [], ""), + ({"serves": False, "router": None}, [], ""), + ], +) +def test_marked_and_router_refuse_or_wait( + keys: dict[str, Any], booting: list[str], refused: str +) -> None: + verdict = _judge([_beat("a", value=_value(**keys))]) + assert verdict.booting == booting + assert (refused in " ".join(_reasons(verdict))) and bool(verdict.refusals) == bool(refused) + + +def test_no_tenancy_mode_yet_is_booting() -> None: + # A pod whose first cell-state read failed: mode '', marked and router null. + verdict = _judge([_beat("a", mode="", value=_value(marked=None, router=None))]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_fingerprint_on_neither_current_nor_next_refuses() -> None: + drifted = {**_value(), **cell.as_value({**FIELDS, "oidc_client_id": "drifted"})} + assert _reasons(_judge([_beat("a", value=drifted)])) == [ + "process a: runs neither loom.cell_self's current nor its next configuration" + ] + + +def test_the_final_refusals() -> None: + scim = {**_value(), **cell.as_value({**FIELDS, "scim": True})} + verdict = beats.judge( + [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" + ) + assert [r.reason for r in beats.final_refusals(verdict)] == [ + "SCIM is on: attach with SCIM on needs the per-cell SCIM release" + ] + two = _judge([_beat("a"), _beat("b", keys_fp="other")]) + assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason + assert "no live process" in beats.final_refusals(_judge([]))[0].reason + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + c = _Clock() + + async def _sleep(s: float) -> None: + c.now += s + + monkeypatch.setattr(apg, "_sleep", _sleep) + return c + + +@pytest.mark.asyncio +async def test_settle_waits_for_a_booting_process(clock: _Clock) -> None: + reads = [[_beat("a", cell_name=None)], [_beat("a", cell_name=None)], [_beat("a")]] + + async def read() -> list[beats.Beat]: + return reads.pop(0) + + wait = apg.Wait(10, clock) + verdict = await beats.settle(read, _judge, wait) + assert (verdict.live, wait.spent) == (1, 2.0) + + +@pytest.mark.asyncio +async def test_settle_refuses_at_the_end_of_wait_naming_what_it_waited_on(clock: _Clock) -> None: + async def read() -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name="", live=False, stopped=True)] + + with pytest.raises(Refused) as e: + await beats.settle(read, lambda f: _judge(f, "wait"), apg.Wait(3, clock)) + assert [str(r) for r in e.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: stopped with LOOM_CELL unset, not aged out within --wait", + ] + + +@pytest.mark.asyncio +async def test_settle_extends_once_for_a_process_first_seen_booting(clock: _Clock) -> None: + seen = 0 + + async def read() -> list[beats.Beat]: + nonlocal seen + seen += 1 + # A stopped process aging out first; then one booting from the third read (2 s + # spent), done at the seventh, past the first deadline. + if seen < 3: + return [_beat("a"), _beat("x", cell_name="", live=False, stopped=True)] + return [_beat("a"), _beat("b", cell_name=None)] if seen < 7 else [_beat("a"), _beat("b")] + + wait = apg.Wait(3, clock) + verdict = await beats.settle(read, lambda f: _judge(f, "wait"), wait, extend_s=4) + assert (verdict.live, wait.seconds) == (2, 6) + + async def forever() -> list[beats.Beat]: + return [_beat("b", cell_name=None)] + + wait = apg.Wait(2, clock) + with pytest.raises(Refused, match="still booting"): + await beats.settle(forever, _judge, wait, extend_s=1) + # Seen booting from the start: first sight plus the extension is inside the deadline. + assert wait.seconds == 2 + + +@pytest.mark.asyncio +async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: + wait = apg.Wait(2, clock) + clock.now += 100 # a scan's time + assert await wait.pause() and await wait.pause() + assert not await wait.pause() + assert wait.spent == 2.0 + + +# -- the rescan's watch ---------------------------------------------------------------- + + +def test_the_prune_contract_is_an_hour() -> None: + assert instances.PRUNE_AFTER_S == 3600 + + +@pytest.mark.asyncio +async def test_the_watch_refuses_every_unset_process_it_saw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rounds = [ + [ + _beat("a", cell_name=""), + _beat("c"), + _beat("d", cell_name=None, live=False, stopped=True), + ], + # a's rows were swept; c's cell row alone was swept: it was seen with the cell. + [_beat("c", cell_name=None, live=False, stopped=True)], + ] + + async def read_beats(_ch: Any, *, since: str | None = None) -> list[beats.Beat]: + assert since == "2026-01-01 00:00:00.000" + return rounds.pop(0) + + monkeypatch.setattr(beats, "read_beats", read_beats) + clock = _Clock() + watch = beats.Watch(None, "2026-01-01 00:00:00.000", "first", clock) + await watch.read_once() + await watch.read_once() + assert [r.subject for r in watch.refusals()] == ["process a", "process d"] + clock.now += beats.watch_period_s() + 1 + monkeypatch.setattr(beats, "read_beats", read_beats) + rounds.append([]) + await watch.read_once() + assert watch.refusals()[-1].subject == "heartbeats" + + +def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) + assert beats.Watch(None, "", "first").refusals_up_front() + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) + assert beats.Watch(None, "", "first").refusals_up_front() == [] + + +# -- batches, ordering, arguments -------------------------------------------------------- + + +def test_batches_keep_the_count_and_the_byte_bound() -> None: + many = [f"org-{i:05d}" for i in range(ATTACH_BATCH + 1)] + assert [len(b) for b in cell_attach.batches(many, "dep")] == [ATTACH_BATCH, 1] + # Escaping grows a non-ASCII name six times: the encoded body is what counts. + wide = ["é" * 2000 + str(i) for i in range(10)] + out = cell_attach.batches(wide, "dep") + assert len(out) > 1 and sum(map(len, out)) == 10 + for b in out: + body = json.dumps({"deployment_id": "dep", "orgs": b}, separators=(",", ":")) + assert len(body.encode()) <= MAX_BODY_BYTES + + +def test_move_sources_are_scanned_first() -> None: + tables = [Table("loom", n, "MergeTree") for n in ("a", "memory_extraction_queue", "b", "src")] + ordered = cell_scan.order_sources(tables, {"loom.src"}) + assert [t.name for t in ordered] == ["memory_extraction_queue", "src", "a", "b"] + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 10, + "scan_budget": 100, + "dry_run": False, + "ch_allow_user": [], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": "http://127.0.0.1:7900", + } + return argparse.Namespace(**{**fields, **over}) + + +def test_the_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") + assert cell_attach._args_problem(_args()) == "" + assert "LOOM_CELL" in cell_attach._args_problem(_args(cell="other")) + assert "--scan-budget" in cell_attach._args_problem(_args(scan_budget=0)) + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + assert "Postgres" in cell_attach._args_problem(_args()) + + +def test_an_org_past_the_limit_is_named_with_its_sources() -> None: + found = cell_scan.ScanResult() + found.add("postgres loom.tenancy", "x" * (cell.ORG_MAX_BYTES + 1)) + (refusal,) = cell_attach._too_long(found) + assert refusal.subject.startswith(f"org of {cell.ORG_MAX_BYTES + 1} bytes") + assert "found in postgres loom.tenancy" in refusal.reason + + +# -- the router ------------------------------------------------------------------------ + + +def _router(*responses: httpx.Response) -> cell_attach.Router: + answers = list(responses) + + def handler(_request: httpx.Request) -> httpx.Response: + return answers.pop(0) + + return cell_attach.Router( + httpx.URL("http://127.0.0.1:7900"), "first", httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_the_router_calls_refuse_as_the_router_answers() -> None: + ok = {"cell": "first", "state": "attaching"} + assert await _router(httpx.Response(200, json=ok)).call("") == ok + with pytest.raises(Refused, match="not upgraded"): + await _router(httpx.Response(404, json={"error": "no route"})).call("/attach/place", {}) + with pytest.raises(Refused, match=r"refused /attach/begin \(409\): use loom cell update"): + await _router(httpx.Response(409, json={"error": "use loom cell update"})).call( + "/attach/begin", {} + ) + with pytest.raises(Refused, match="did not answer as the router does"): + await _router(httpx.Response(502, text="bad gateway")).call("") + + +# -- the Postgres reads, on a fake session --------------------------------------------- + + +@dataclass +class FakeConn: + """Answers by the first needle the SQL holds; a callable answer is called with the + arguments; an exception answer is raised.""" + + routes: list[tuple[str, Any]] = field(default_factory=list) + executed: list[str] = field(default_factory=list) + + def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else answer + raise AssertionError(f"unrouted: {sql[:80]}") + + async def fetch(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchrow(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchval(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def execute(self, sql: str, *args: Any) -> str: + self.executed.append(sql) + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else str(answer) + return "OK" + + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + + return _Tx() + + +def _seq(*values: Any) -> Callable[..., Any]: + pending = list(values) + return lambda *_a: pending.pop(0) + + +@pytest.mark.asyncio +async def test_the_session_checks_refuse_a_pooler() -> None: + moved = FakeConn( + [ + ("pg_backend_pid() AS pid", _seq((1, "t0"), (2, "t1"))), + ("current_setting($1, true)", None), + ] + ) + refusals = await apg.direct_refusals(moved, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + "attach's session moved between backends", + "attach's session loses loom.tenancy_checked", + ] + direct = FakeConn([("pg_backend_pid() AS pid", (1, "t0")), ("current_setting($1, true)", "on")]) + assert await apg.direct_refusals(direct, "attach") == [] + assert "RESET ALL" in direct.executed + + +@pytest.mark.asyncio +async def test_the_lock_refuses_when_held_or_not_on_this_backend() -> None: + holder = [{"pid": 7, "application_name": "loom", "client": "10.0.0.1", "backend_start": "t"}] + held = FakeConn([("pg_try_advisory_lock", False), ("FROM pg_locks l LEFT JOIN", holder)]) + with pytest.raises(Refused, match="held by pid 7 \\(loom, from 10.0.0.1"): + await apg.take_lock(held, "attach") + pooled = FakeConn([("pg_try_advisory_lock", True), ("SELECT EXISTS", False)]) + with pytest.raises(Refused, match="a pooler"): + await apg.take_lock(pooled, "attach") + assert any("tcp_keepalives_idle" in s for s in pooled.executed) + + +@pytest.mark.asyncio +async def test_a_database_not_utf8_refuses() -> None: + assert await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "UTF8")])) == [] + (r,) = await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "SQL_ASCII")])) + assert "SQL_ASCII, not UTF8" in r.reason + + +def test_privilege_holders_outside_the_allowed_refuse() -> None: + holders = apg.Holders( + own=["loom"], + others={"ci": ["may insert into loom.tenancy"], "servers": ["owns loom.memory_current"]}, + ) + assert [r.subject for r in apg.privilege_refusals(holders, ["servers"])] == ["postgres role ci"] + assert apg.privilege_refusals(holders, ["ci", "servers"]) == [] + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_refuse() -> None: + conn = FakeConn( + [ + ( + "pg_db_role_setting", + [{"role": "ci", "db": "every database", "cfg": "Loom.Tenancy_Checked=on"}], + ), + ("pg_subscription", [{"subname": "mirror", "subenabled": False}]), + ] + ) + refusals = await apg.unmarked_refusals(conn, True) + assert [r.subject for r in refusals] == [ + "postgres unmarked session", + "postgres setting for ci in every database", + "postgres subscription mirror", + ] + + +@pytest.mark.asyncio +async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: + conn = FakeConn( + [ + ("SELECT current_user", "attacher"), + ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), + ("max_prepared_transactions", 5), + ] + ) + refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + assert [str(r) for r in refusals] == [ + "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " + "session, so step 4 could not wait for it; set it to 0", + ] + + +def _row(**over: Any) -> dict[str, Any]: + return { + "pid": 9, + "backend_type": "client backend", + "application_name": "ci", + "xact_start": "t", + "ours": True, + "xid_old": False, + "xmin_old": False, + "hidden": False, + **over, + } + + +@pytest.mark.asyncio +async def test_step_4_refuses_what_it_cannot_wait_for(clock: _Clock) -> None: + base = [("max_prepared_transactions", 0), ("pg_prepared_xacts", 0)] + hidden = FakeConn([*base, ("pg_stat_activity s", [_row(hidden=True)])]) + with pytest.raises(Refused, match="hidden from attach's role"): + await apg.wait_transactions(hidden, 100, apg.Wait(5, clock)) + prepared = FakeConn([("max_prepared_transactions", 0), ("pg_prepared_xacts", 1)]) + with pytest.raises(Refused, match="prepared transaction"): + await apg.wait_transactions(prepared, 100, apg.Wait(5, clock)) + held = FakeConn( + [ + *base, + ( + "pg_stat_activity s", + [_row(xmin_old=True), _row(pid=10, ours=False, xid_old=True), _row(pid=11)], + ), + ] + ) + with pytest.raises(Refused) as e: + await apg.wait_transactions(held, 100, apg.Wait(2, clock)) + assert [r.reason for r in e.value.refusals] == [ + "may have read `recording`: still open", + "another database's transaction, holding the snapshot horizon", + ] + done = FakeConn([*base, ("pg_stat_activity s", [_row(pid=10, ours=False, xid_old=True)])]) + await apg.wait_transactions(done, 100, apg.Wait(0, clock)) + + +class _Timeout(Exception): + sqlstate = "55P03" + + +@pytest.mark.asyncio +async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: + tries = _seq(_Timeout(), "UPDATE 1") + + def lock(*_a: Any) -> str: + answer = tries() + if isinstance(answer, Exception): + raise answer + return "LOCK TABLE" + + holders = _seq( + [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] + ) + conn = FakeConn( + [ + ("LOCK TABLE", lock), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", holders), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + confirmed: list[bool] = [] + + async def confirm() -> None: + confirmed.append(True) + + async def ready() -> bool: + return True + + logged: list[str] = [] + await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + assert logged == ["the mode's lock is busy; retrying"] + # Before each try and the one termination. + assert len(confirmed) == 3 + + +@pytest.mark.asyncio +async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: + conn = FakeConn( + [ + ( + "l.mode = 'ExclusiveLock'", + [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], + ), + ("rolsuper FROM pg_roles", False), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused, match="a superuser must end it"): + await apg.end_mode_transactions(conn, confirm) + + +@pytest.mark.asyncio +async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( + clock: _Clock, +) -> None: + conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + + async def confirm() -> None: + pass + + async def booting() -> bool: + return False + + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + assert not any("LOCK TABLE" in s for s in conn.executed) + + +@pytest.mark.asyncio +async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) + + async def read(mode: str, xid: int) -> apg.Durable: + return apg.Durable(mode=mode, first=(True, "first"), xid=xid) + + monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + with pytest.raises(Refused, match="did not land"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + with pytest.raises(Refused, match="is not above 100"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + assert await run.durable() == 100 + + async def waiting() -> str: + return "waiting" + + monkeypatch.setattr(pg_profile, "standby_now", waiting) + with pytest.raises(Refused, match="not synchronous before"): + await run.durable() + + +@pytest.mark.asyncio +async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + state = ["enforced"] + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return state[0], (True, "first") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "read_mode", read_mode) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + assert await run.done() + state[0] = "recording" + with pytest.raises(Refused, match="run loom cell restore"): + await run.done() + + +def test_refusal_text() -> None: + assert str(Refusal("a", "b")) == "a: b" diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 3f670d4014..decb365521 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -695,7 +695,7 @@ def test_doctor_runs_the_cell_checks_only_with_loom_cell( monkeypatch.setattr(doctor, "_doctor_scim_token_check", lambda _s: ("scim_token", "PASS", "")) called: list[bool] = [] monkeypatch.setattr( - doctor_cell, "cell_checks", lambda: called.append(True) or [("cell_state", "PASS", "")] + doctor_cell, "cell_checks", lambda *_: called.append(True) or [("cell_state", "PASS", "")] ) names = [n for n, _g, _d in doctor._run_live_checks()] assert called == ([True] if runs else []) and ("cell_state" in names) == runs diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index e9ea657df6..c978eb94e2 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import contextlib import json from datetime import UTC, datetime from typing import Any @@ -29,6 +30,8 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). + monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) def _args(**over: Any) -> argparse.Namespace: diff --git a/tests/unit/test_doctor_client_checks.py b/tests/unit/test_doctor_client_checks.py index 554220a424..a2e1b0bd01 100644 --- a/tests/unit/test_doctor_client_checks.py +++ b/tests/unit/test_doctor_client_checks.py @@ -572,7 +572,7 @@ def test_doctor_client_flag_appends_the_lines_and_json_carries_them( ) -> None: from server.cli import doctor - monkeypatch.setattr(doctor, "_run_doctor_checks", lambda: [("python", "PASS", "3.12")]) + monkeypatch.setattr(doctor, "_run_doctor_checks", lambda *_: [("python", "PASS", "3.12")]) monkeypatch.setattr(doctor, "_doctor_problems", lambda: []) monkeypatch.setattr( doctor, diff --git a/tests/unit/test_doctor_client_client_only.py b/tests/unit/test_doctor_client_client_only.py index 60ceb323d6..712c10d7cd 100644 --- a/tests/unit/test_doctor_client_client_only.py +++ b/tests/unit/test_doctor_client_client_only.py @@ -56,7 +56,7 @@ def test_clickhouse_line_names_the_probed_host_on_failure(monkeypatch: pytest.Mo assert detail.startswith("https://ch.example.com:8443 → TimeoutError") -def _live() -> list[Check]: +def _live(*_roles: Any) -> list[Check]: return [ ("python_version", "PASS", "3.12"), ("clickhouse_reachable", "FAIL", "https://ch.example.com:8443 → TimeoutError: timed out"), From 6cf53b3c58ca49427c3e7ecfe4a8e5d33823ec62 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:19:41 +0000 Subject: [PATCH 002/181] Attach's runbook, the cells.md first-attach steps and the trigger-cost gate measured end to end docs/operations/cell-attach.md is the operator's procedure: the preconditions attach checks and those it relies on, the flags, the point of no return at the mode commit, the orgs attach does not place and the manual removal of an org past the length limit, and doctor's reads after serving. cells.md describes the first attach as it runs (the lock, the refusals, the trigger pass, the mode commit and its durable read, step 4's wait, rescan and final checks), with its stated limits and failure rows. upgrades.md step 18 rotates the ClickHouse writer credential after the LOOM_CELL rollout. bench/cell_trigger_gate.py measures memory.set and a delivery-sized insert through an attached cell with the org triggers on and off; results.md records a local run. Attach compares the final root-secret fingerprint with the one its begin wrote, and the nonce cleanup's message names no single command. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 117 ++++++++++++ docs/architecture/cells.md | 290 +++++++++++++++++++---------- docs/benchmarks/results.md | 31 ++- docs/operations/cell-attach.md | 148 +++++++++++++++ docs/operations/postgres.md | 4 + docs/operations/upgrades.md | 8 + docs/reference/cli.md | 3 +- llms.txt | 1 + server/cli/cell_attach.py | 10 +- server/cli/cell_prepare.py | 2 +- tests/unit/snapshots/cli_help.json | 12 +- tests/unit/snapshots/cli_tree.json | 261 +++++++++++++++++++++++++- tests/unit/test_cell_attach.py | 17 ++ 13 files changed, 791 insertions(+), 113 deletions(-) create mode 100644 bench/cell_trigger_gate.py create mode 100644 docs/operations/cell-attach.md diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py new file mode 100644 index 0000000000..e90fbb32fa --- /dev/null +++ b/bench/cell_trigger_gate.py @@ -0,0 +1,117 @@ +"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server +of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an +unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` +guard stays). Run it against an attached cell's server, which must not serve anything else, +with a writer token for a placed org; it drops and recreates the cell's org triggers, so +never against a deployment in use. Needs no provider. + + PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ + --base-url http://127.0.0.1:7932 --pg-dsn \\ + --token --org --namespace +""" + +from __future__ import annotations + +import argparse +import asyncio +import statistics +import time +import uuid + +import asyncpg +import httpx + +from server.core import cell_triggers + +CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 + + +async def _triggers(dsn: str, on: bool) -> None: + """Create or drop the org triggers, and check that what was asked holds: every guarded + table carries both triggers enabled at origin, or none does.""" + conn = await asyncpg.connect(dsn) + try: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + for table in guarded: + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + enabled = await conn.fetchval( + "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", + list(cell_triggers.ORG_TRIGGERS), + ) + finally: + await conn.close() + expected = 2 * len(guarded) if on else 0 + if enabled != expected: + raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + + +async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: + """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" + lat = [] + headers = {"Authorization": f"Bearer {args.token}"} + async with httpx.AsyncClient(timeout=30, headers=headers) as client: + for i in range(CALLS + WARMUP): + body = { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": f"gate-{uuid.uuid4().hex[:10]}", + "value": f"trigger gate {i}", + } + t = time.perf_counter() + (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() + if i >= WARMUP: + lat.append((time.perf_counter() - t) * 1000) + lat.sort() + return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] + + +async def _batch(args: argparse.Namespace, marked: bool) -> float: + """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" + settings = {"loom.tenancy_checked": "on"} if marked else {} + conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + try: + times = [] + for _ in range(INSERTS): + t = time.perf_counter() + await conn.execute( + "INSERT INTO loom.scope_generations (org, scope) " + "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + args.org, + uuid.uuid4().hex[:8], + ) + times.append((time.perf_counter() - t) * 1000) + return statistics.median(times) + finally: + await conn.close() + + +async def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--base-url", required=True) + p.add_argument("--pg-dsn", required=True) + p.add_argument("--token", required=True) + p.add_argument("--org", required=True) + p.add_argument("--namespace", required=True) + args = p.parse_args() + try: + for _ in range(ROUNDS): + for on in (True, False): + await _triggers(args.pg_dsn, on) + p50, p99 = await _memory_set(args) + marked, unmarked = await _batch(args, True), await _batch(args, False) + print( + f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " + f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " + f"{unmarked:.1f} ms unmarked", + flush=True, + ) + finally: + await _triggers(args.pg_dsn, True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..96580ec7d2 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -389,7 +389,8 @@ open meanwhile, as on a deployment never prepared, since the cell's Postgres, wh holds its state, cannot tell the two apart (ClickHouse carries no cell state: adding it would make the write path read a second store for tenancy) and such a process never reads the directory; where every process has `LOOM_CELL` unset, `loom doctor`'s naming -of it (which reads the directory, and ships with attach) is the only signal. A +of it (which reads the directory, and ships with the router's read-only scope) is the only +signal. A `(true, …)` row with `recording` and the own row: the first cell recording. Both present with the own row otherwise: `enforced`, a later cell's mode reading as `enforced` whatever its row says (its triggers, mode cache and CLI treat it so, and @@ -676,20 +677,18 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after any error in that -transaction it ends that session's backend (`pg_terminate_backend`, the backend named by -the pid and `backend_start` attach captured when it connected) and waits until it has -left `pg_stat_activity`, then reads the mode back before it retries or reports; a mode -read back as `enforced` is written again in a fresh transaction, whose commit waits for -the standby, before attach goes on, since a commit ended while it waited is on the -primary alone. The attach command fixes those timeouts, and its tests cover each of -these cases. The mode cache reads the mode with a plain `SELECT`, which takes no row -lock and never waits on attach. An insert that finds `enforced` (a process whose cache +server-side timeouts, then writes the mode and commits; after a failed try it ends the +mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in +this database (a pid captured at connect is wrong behind a connection path that swaps +backends), and waits until the lock clears. Whether the commit landed is decided by a +durable read of the mode, whose commit waits for the standby, since a commit ended while +it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never +waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in -`enforced`, and a CLI command refuses as in `enforced`, naming the org. A process whose -cache still reads `enforced` after step 3 or step 4 returns the cell to `recording` places -through the router's gate until its cache expires. -`loom doctor` lists any tenancy row the directory does not place. +`enforced`, and a CLI command refuses as in `enforced`, naming the org. A first attach +never lowers the mode once it commits `enforced`. `loom doctor`, with the router's +read-only scope, lists any tenancy row the directory does not place. - **It runs after verification, before anything is written for the org.** The op gate (`gates.authorize`) checks the org the transport resolves (`gates.resolve_org`, with @@ -983,51 +982,126 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. -The first attach turns the existing deployment into the first cell, in order: - -1. check the mode is `recording`, as prepare set it: every verified request has written - its org's tenancy row if absent while the directory answered that the cell is - `attaching`; -2. in one directory transaction, write `attach_started_at`, the cell's issuers, patterns - and fingerprint (both values of `loom.cell_self`), and the placements, then the - tenancy rows, of every org the cell holds, from one snapshot, straight into the - directory (no other cell exists yet): its configured orgs (of both values), the - distinct `org` of every table of both schemas (generated from the schema, as - erasure's table classification is), and the orgs of catalogued bucket objects; never - `''`, `'-'` or `'*'`; -3. set `enforced` as above, within `--wait`, which the wait below then shares, starting - no try that could commit the mode with less left than that wait's minimum (the mode - cache's lifetime plus the longest request deadline): attach then refuses, reporting - the mode it reads back (`recording`, and a rerun resumes from step 1), or the mode as - unknown when it cannot read it, and a rerun then resumes by the mode it reads; then - wait, within what remains of `--wait` (default 10 minutes, then refuse and set - `recording` back), until every live process reports `enforced` and then the mode - cache's lifetime plus the longest request deadline have passed; then run the SCIM - precondition again (`--revoke-unprovisioned` again if given), for accounts made - before `enforced` took hold; if it refuses, the cell stays `enforced` and - `attaching`, and running attach again resumes from the wait, as a resumed attach does - (below); -4. wait, within what remains of `--wait`, until no transaction that could have read - `recording` is still open, judged from a Postgres snapshot attach takes once it has - read `enforced` back in this run (not from a clock): a transaction on the cell's - database that started before it, from any client, background worker or logical - replication connection, read-only ones included, a writing transaction on another - database of the same server that did (it holds back every snapshot's horizon, and is - named as such), a prepared transaction (which refuses at once), or one whose state - attach cannot read counts as open, and Postgres's own maintenance processes, physical - replication connections (every standby's and a base backup's, which run no SQL) and - attach's own connections do not. Past `--wait` attach refuses, naming what it waited - on, and leaves the cell `enforced`, so a rerun resumes from step 3's wait. The attach - command fixes the exact query, and its tests cover each kind of transaction. Once the - wait passes, place the tenancy rows written since the snapshot and every org step 2's - sources yield when read again (the distinct `org` of every table of both schemas and - the orgs of catalogued bucket objects; a row or an object an unchecked writer, such - as an older CLI or a worker, wrote in `recording` after the snapshot is placed too), - check again that every live process reports `enforced` (refusing and returning to - `recording` as step 3 does if one does not), and mark the cell `serving`. The - directory records the highest mode attach has raised or tried to raise, which - `loom cell restore` writes back only upward: a cell whose step 3 was refused at the - lock is restored `enforced`, and its resumed attach then runs from step 3's wait. +The first attach (`loom cell attach`, its procedure on +[attaching the first cell](../operations/cell-attach.md)) turns the existing deployment +into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at +`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and +`loom cell update` take the same lock), after checking that the session is one direct +backend that keeps the tenancy marker: a lock left on a pooler's server connection would +refuse every later run, and attach's tenancy rows would be refused as an unchecked +writer's. A lock found lost between steps is taken again (refusing when another run holds +it), and the cell's state is read again. The lock does not make a run safe, since every +step is idempotent and the mode only rises; it keeps a second run's scans from holding the +snapshot horizon that step 4 waits on. + +It refuses, before any write, naming each: + +- stores other than the registered ones: the Postgres deployment id, ClickHouse's + `loom.storage_authority` and the shared host's `/v1/health/deployment` each name the + directory row's deployment id, `loom.cell_first` reads `(true, )`, the row is the + first cell's, `attaching`, with no other directory row; +- a database encoding other than UTF8 (prepare refuses the same), so a tenancy row + within the length limit is always an org a request can name; +- a synchronous standby not attached now, a durability profile that would not refuse + without one, or a role that may not write the durable read's logical message; a role + of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with + `max_prepared_transactions` above 0; +- **the tenancy-privilege check**: a login role, other than attach's own (superusers, + and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and + those named with `--pg-allow-role`, that holds or reaches through any membership a right + to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or + `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, + `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system + one (a function there can shadow one the servers call unqualified on their marked + session), `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` on + `loom.tenancy_checked`, membership in `pg_write_server_files` or + `pg_execute_server_program`, or ownership of a `loom` table, of the trigger functions or + of `loom.cell_first_immutable()`, the database or a schema; +- **the unmarked-session check**: a session that sends no marker reads one, or runs as + `replica`; a `pg_db_role_setting` entry of `loom.tenancy_checked` or + `session_replication_role` for this database or every one, matched case-insensitively; + any logical-replication subscription into this database, enabled or not (its apply + worker fires no origin trigger); +- the heartbeats (both tables, stopped rows included for the widened window, the live + window plus a stopping pod's drain and retirement): a live process with `LOOM_CELL` + unset or another cell, one without the heartbeat's keys, `marked` false, `router` false + where `serves` is true, a fingerprint on neither `loom.cell_self` value, several + `keys_fp` values, or none live; a booting process (no cell row yet, no mode, `marked` + or `router` null) is waited on within `--wait`; in `recording` a stopped process with + `LOOM_CELL` unset refuses too; +- the ClickHouse writer check and the scan's table list (`server/cli/cell_writers.py`, + `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user + reads. + +Then, in order: + +1. `attach/begin`, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, + through `attach/enforced` again, to the durable read. +3. Create the triggers itself, on every run, before step 2's scan and again just before the + mode commit (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads + every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an + `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with + the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only + org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such + orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row + is written; a name no request can carry is skipped and named. Then the tenancy rows of + the orgs found, on a marked session, and no placement: unscoped callers are answered + from the tenancy rows, and placing now would move the point of no return earlier. +5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of + `--scan-budget`, since step 4 reads the same tables again. Each refuses in + `recording`, which can still be backed out. +6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on + `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock + (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database + alone) and ended. From the commit on the mode only rises: every later refusal leaves + the cell `enforced` and `attaching`, and a rerun resumes from the durable read. +7. The durable read, in every run that reads `enforced`: the mode and the `first` row read + in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. + Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED + writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a + snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is + assigned after the mode commit. +8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends + on its cached mode (a request records a new org through `tenancy._RECORD`, which reads + the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling + one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A + prepared transaction, or a row of the cell's database whose state is hidden, refuses + at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the + heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); + the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery + tables move rows to ClickHouse), and the queue and drain tables and the tables a view + reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, + in `attach/place` batches; the writer check again; and the final checks, under a floor + of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): + the heartbeats, read through the rescan every half of the time the leader keeps a + stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` + unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` + equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 +hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would +be placed and skipped and the scan's time, and writes nothing; the router's checks at +`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is +done (exit 0), anything else refuses, naming `loom cell restore`. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1046,13 +1120,18 @@ cell with SCIM on admits no one until its first provisioning cycle has run. A ce nearing its stores' limits is closed to new orgs: its placement calls answer 503, so its issuers' new orgs wait for capacity rather than spill elsewhere. -A resumed first attach, for a cell whose directory row has `attach_started_at`, reruns -every refusal of the list above that applies to the first cell. It then continues by the -cell's mode: in `recording` from step 1; in `enforced`, from step 3's wait (every live -process reports `enforced`, then the cache's lifetime and the longest request deadline -pass) and then its SCIM precondition, so the SCIM check that ends an attach stopped -after setting `enforced`, whatever stopped it, always runs after `enforced` has taken -hold. +A resumed first attach reruns every refusal and continues by the directory row and the +cell's mode: in `recording` from the scan, in `enforced` from the durable read; whenever +the cell reads `enforced` it posts `attach/enforced` again, so a directory restored below +the cell's mode is raised. + +The checks of the stores' writers are point-in-time: a grant, a role setting, a +subscription, a ClickHouse writer or object added after `serving` passes them, as does a +configuration file attach cannot read (`postgresql.conf`, `postgresql.auto.conf`, a +standby's own) or a role that runs code as the server's OS user. `loom doctor` runs the +tenancy-privilege and unmarked-session reads again on every run and warns on what it finds +(`cell_tenancy_privilege`, `cell_unchecked_writers`); the runbook has the operator confirm +the rest. ## What each cell runs @@ -1102,9 +1181,9 @@ closed: every cell keeps serving the orgs it holds, routing continues from each replica's signed snapshot of its last cache (written to shared storage on every directory change), and the placements, lookups of uncached orgs, `loom cell` commands that read or write the directory and the recording cells' state reads answer 503 or -refuse until the directory is back, while `loom doctor` reports its directory checks as -not run and, with `LOOM_CELL` set, exits non-zero; rebuilding a lost directory from the -cells is a design of its own. +refuse until the directory is back, while `loom doctor`, with the router's read-only +scope, reports its directory checks as not run and, with `LOOM_CELL` set, exits +non-zero; rebuilding a lost directory from the cells is a design of its own. Each cell keeps, in its own Postgres, a copy of its directory row's configuration fingerprint (`loom.cell_self`), which its processes compare their configuration with and @@ -1200,30 +1279,36 @@ the directory row. ## Failure modes -| Failure | What happens | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | -| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | -| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | -| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | -| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | -| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | -| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | -| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | -| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | -| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | -| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | -| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | -| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | -| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | -| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | -| A recording insert held open during first attach | attach stops trying once too little of `--wait` is left for the wait, reporting the mode it reads back; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | -| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | -| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach | -| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| Failure | What happens | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | +| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | +| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | +| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | +| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | +| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | +| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | +| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | +| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | +| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | +| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | +| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | +| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | +| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | +| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | +| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | +| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | +| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | +| A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | +| `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | +| A process that changes between attach's last check and the mode commit, or an HA manager that empties `synchronous_standby_names` between the checks around the durable read | not seen by attach: a window of seconds | +| A cell process with a longer `LOOM_INSTANCE_HEARTBEAT_SECONDS` than attach's | it can fall outside attach's live window while live; the runbook requires one value | ## Rolling upgrades @@ -1306,11 +1391,16 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). `loom doctor` names), with their functions in `schemas/pg_migrations/011_tenancy_triggers.sql` and `012_tenancy_guard.sql`; `server/cli/doctor_cell.py`: `loom doctor`'s cell checks; -- `server/cli/cell_scan.py`: the first attach's org scan, a library no command calls: the - tables of both stores with an `org` column, judged by engine and against erasure's - classification, every host of a cluster listing the same, and the distinct orgs of each - table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the - users, user directories, objects and replicas outside the operator's allow lists); +- `server/cli/cell_attach.py`: `loom cell attach`; `server/cli/cell_attach_pg.py`: its + Postgres reads and writes (the session checks, the attach lock, the tenancy-privilege + and unmarked-session checks doctor shares, the trigger pass, the mode commit, the + durable read, step 4's wait); `server/cli/cell_attach_beats.py`: its judgment of the + heartbeats and the rescan's watch; +- `server/cli/cell_scan.py`: the first attach's org scan: the tables of both stores with + an `org` column, judged by engine and against erasure's classification, every host of a + cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; + `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, + objects and replicas outside the operator's allow lists); `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; @@ -1386,14 +1476,14 @@ An integration test with two cells and a router in CI's Postgres job: CLI censuses cover the rest); - the first attach places every org with a row in any table, an org first seen during the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's snapshot and `enforced`, a ClickHouse row an older CLI wrote then and a + step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer request, passes its health probe, answers the probe route and runs no worker on an attached cell and on a later cell between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included after step 3 or step 4 returned the cell to it; a CLI command + `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token revocation) runs, also with Postgres down on a deployment never prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 07367d3688..81e099d3c8 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1220,8 +1220,35 @@ a row for the state read and the `loom.tenancy` lookup. Loom's own writes are ma this predicts the ship gate (a `memory.set` p99 rise of at most 10 ms, a delivery pass at most 10% slower) holds by a wide margin; the cost lands on scripts and older builds writing a cell without the check. This measures the triggers on the statements, not a -whole `memory.set` through a server on a cell: the gate itself is measured end to end -with attach. +whole `memory.set` through a server on a cell; the end-to-end gate follows. + +`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached +by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous +standby and ClickHouse in Docker on a shared host under other load), 400 sequential +`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert +of the held org from a marked and an unmarked session, the org triggers on and dropped in +turn, three rounds each; the harness checks the triggers it asked for are enabled: + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ + --pg-dsn --token --org --namespace +``` + +| Local, three rounds | Triggers on | Triggers off | +| ----------------------------------- | ---------------------- | ---------------------- | +| `memory.set` p50 | 154.4, 99.3, 98.4 ms | 91.9, 101.2, 101.6 ms | +| `memory.set` p99 | 494.0, 291.7, 257.6 ms | 279.2, 244.7, 233.3 ms | +| 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | +| 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | + +The marked batch, which is what a delivery pass of a cell's own processes writes, shows no +cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. +The `memory.set` figures move by more between rounds of one configuration than any trigger +cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms +here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no +consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the +host's noise, and the statement-level figures above bound the marked cost at under a +microsecond a row. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md new file mode 100644 index 0000000000..12a0cbe247 --- /dev/null +++ b/docs/operations/cell-attach.md @@ -0,0 +1,148 @@ +# Attaching the first cell + +`loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, +directory row `attaching`) into a `serving` one: it places every org the cell holds in the +router's directory, raises the cell's mode to `enforced` and creates its org triggers. This +page is the operator's procedure; the design is +[cells](../architecture/cells.md#attaching-a-cell). + +## Before attach + +Run attach at low load, as one run at a time, and not during a rollout, a provider +transition, a logical backup, a long erasure or a migration: each holds a transaction or a +snapshot long enough for step 4's wait to run out of `--wait`. Steps 2 and 4 run index skip +scans on the Postgres primary (each statement holds vacuum's horizon while it runs) and +read the `org` column of every ClickHouse table, which competes with serving load. + +Every item below is a precondition attach checks, or one it cannot check and relies on: + +1. Every router replica runs this release; every process and every CLI and CI image of + the cell runs it too ([upgrades](upgrades.md#upgrade-procedure), steps 16 and 17). +2. `LOOM_CELL` is set on every process of the cell. Every CLI and CI configuration that + writes for an org has `LOOM_CELL` set without `LOOM_CELL_ROUTER_TOKEN`: in `recording` + it then refuses only new orgs, so nothing has to change at the mode commit. With the + token removed, revoke the `INSERT` on `loom.tenancy` and the `UPDATE (since)` on + `loom.tenancy_mode` the CI role held for recording new orgs. +3. The ClickHouse writer credential is rotated after that rollout + ([upgrades](upgrades.md#upgrade-procedure), step 18). +4. No fingerprinted setting has changed since prepare, and none changes until attach ends: + a change gates every process against `loom.cell_self`. +5. `LOOM_SERVES_TRAFFIC=false` is set only on pods no load balancer routes to: attach + exempts them from the router check. +6. The cell's processes and the attach CLI share one `LOOM_INSTANCE_HEARTBEAT_SECONDS`: + attach judges which processes are live by its own value. +7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers + and for attach: attach refuses a session that moves between backends or loses the + tenancy marker, and refuses a server whose heartbeat reports `marked` false. +8. SCIM is off: attach refuses with it on. +9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the + cluster has an interserver `` or a `remote_servers` user equal to attach's. + One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an + availability cost, not a safety one. +10. Only Loom's own roles and superusers hold the rights attach's tenancy-privilege check + counts (inserting a tenancy row, changing the mode, creating in a schema, triggers on + `loom` tables, owning them or their trigger functions, `session_replication_role`, + `ALTER SYSTEM` of `loom.tenancy_checked`, `pg_write_server_files`, + `pg_execute_server_program`), and no role that holds one, or owns what it counts, owns + a `SECURITY DEFINER` function another role can execute that does any of that. Attach + checks the catalog; it cannot read a function's body for intent, or a role that runs + code as the server's OS user. +11. `postgresql.conf`, `postgresql.auto.conf` and any pending `ALTER SYSTEM` entry, on the + primary and on every standby, set neither `loom.tenancy_checked` nor + `session_replication_role`. Attach sees a live entry on the primary (its unmarked + session reads it), never a standby's files. +12. Postgres has a synchronous standby attached (`synchronous_standby_names` set) and + `max_prepared_transactions = 0`; the database encoding is UTF8. +13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. + +Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +`loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, +`loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on +`loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role +of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the +writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT +QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings +profile or quota capping the rows or bytes it reads, and the insert into +`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the +bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. + +## Run it + +```bash +loom cell attach --cell --dry-run \ + --ch-allow-user --pg-allow-role +loom cell attach --cell \ + --ch-allow-user --pg-allow-role +``` + +`--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs +per source table, the ones it would skip, and whether the scan fit in half of +`--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object +and replica that may write the configured databases with `--ch-allow-user`, +`--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every +Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each +list is audited by the router. `--wait` (default 600 s) bounds the waits together; +`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of +it, since step 4 reads the same tables again. + +A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a +rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it +is done. + +## The point of no return + +Step 3's mode commit is one-way. Before it, a refused attach leaves the cell `recording`, +its tenancy rows and its triggers in place (they pass every row in `recording`), and the +cell can be backed out (the first-cell branch of +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated)): +`loom cell remove` serves only a first cell that never left `recording`. After it, scoped +principals place their orgs live within seconds, every refusal leaves the cell `enforced` +and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unsetting +`LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` +always goes forward. + +From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account +routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an +ordinary mint scoped to its org places the org on its first request. A direct call of the +router's operator place route, or a live placement with the cell's credential, cannot read +the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's +credential is the servers' alone. + +## Orgs attach does not place + +- **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is + written. One first found in step 4 is skipped and named. Remove such an org by hand + until the stray-org erase command exists, in this order: its bucket objects, from the + keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no + memory content is left outside erasure; then its rows in every table with an `org` column + of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables + included; then its `loom.tenancy` row, on a session with `loom.tenancy_checked` set. +- **A name no request can carry** (not UTF-8, a NUL) found in a data table is skipped and + named; its rows stay outside placement. +- **An org found only in a log table** (`agent_events` and the tables derived from it, the + code search trace) is never placed: a log row keeps the org an unauthenticated caller + typed. Once that org is placed in another cell, its log rows here are outside its + erasure there. + +`loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, +smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left +unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available +yet. + +## After attach + +`loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres +role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an +org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or +a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a +writer added after `serving` passes attach's checks, which ran once; doctor is how it is +found. + +## Related + +- [Cells](../architecture/cells.md) +- [Troubleshooting: cells](troubleshooting.md#cells) +- [Upgrades](upgrades.md) diff --git a/docs/operations/postgres.md b/docs/operations/postgres.md index d087a158c4..895086c474 100644 --- a/docs/operations/postgres.md +++ b/docs/operations/postgres.md @@ -44,6 +44,10 @@ pooler refuses or drops (its org triggers then refuse Loom's own writes for an o cell holds no row for, and the `loom.tenancy` guard its tenancy row for each newly placed org), and `loom doctor` names a session that lost it. Scripts and bulk loaders never set `session_replication_role = replica` on a cell: it skips the org triggers and the guard. +No role is granted `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` +on `loom.tenancy_checked`, or membership in `pg_write_server_files` or +`pg_execute_server_program` on a cell: `loom cell attach` refuses such a role, and +`loom doctor` warns on one ([attaching the first cell](cell-attach.md)). Every replica of the deployment, and its worker process, sets the same values. The full list, with the admission and delivery settings, is in the diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8caf8c789d..50dd5fc46a 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -320,6 +320,14 @@ checkout. answers them from one read per replica per `LOOM_ROUTER_DIRECTORY_CACHE_S` ([cells](../architecture/cells.md#the-heartbeats-keys)). +18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and + CI configuration of the cell (step 16), rotate the ClickHouse credential they write + with: a new user or password for the cell's servers and CLIs, the old one dropped. No + CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` + unset; the rotation cuts off any the rollout missed before it can write an org's rows + past attach's last scan. Name the new user with `--ch-allow-user` + ([attaching the first cell](./cell-attach.md)). + While a migration applies, the old build keeps serving: every migration is additive. The rules that let the two builds serve side by side are in [rolling upgrades](../architecture/rolling-upgrades.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..12bac9a65b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -315,7 +315,7 @@ python scripts/codegen/docs_reference.py | `loom bot create` | Register a bot: its namespaces, and a key (--generate-key or --public-key) and/or a CI trust (--issuer, --audience, --claim). | `--name`, `--org`, `--namespace-role`, `--generate-key`, `--public-key`, `--issuer`, `--audience`, `--claim`, `--jwks-uri`, `--max-token-age`, `--max-access-token-ttl`, `--max-agents`, `--agent-claim`, `--platform-admin`, `--replace` | | `loom bot list` | Every bot, active or disabled. | | | `loom bot disable` | Refuse its proofs and revoke the tokens it holds. | `--name` | -| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--token` | +| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--pg-allow-role`, `--stray-orgs`, `--stray-orgs-budget`, `--token` | | `loom report` | Print the bug form's Environment field: versions and the `loom doctor` checks. | `--open` | | `loom drops` | Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. | `--namespace`, `--org`, `--hours` | | `loom namespaces-gc` | List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per… | `--org`, `--prefix`, `--namespace`, `--exclude-prefix`, `--idle-hours`, `--include-registered`, `--erase`, `--no-purge`, `--skip-external`, `--batch`, `--show`, `--requester`, `--worker-pass` | @@ -330,6 +330,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index ca465459d3..43b8421b0e 100644 --- a/llms.txt +++ b/llms.txt @@ -104,6 +104,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 368f74ebf8..c8b72fb687 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -1,6 +1,6 @@ """`loom cell attach`: turn a prepared first cell into a `serving` one (docs/architecture/cells.md, "Attaching a cell"; the runbook is -docs/operations/cells.md). +docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: @@ -121,7 +121,7 @@ def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: Refusal( f"org of {len(org.encode())} bytes ({org[:64]}…)", f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " - "remove it first (docs/operations/cells.md, the over-long org procedure)", + "remove it first (docs/operations/cell-attach.md, the over-long org procedure)", ) for org, sources in sorted(found.too_long.items()) ] @@ -174,7 +174,8 @@ def __init__( self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) - self.keys_fp = "" + self.keys_fp = "" # the live processes' one root-secret fingerprint, last read + self.begun_keys_fp = "" # the one this run's `attach/begin` wrote self.report: dict[str, Any] = {"cell": self.name} # -- the lock session -------------------------------------------------------------- @@ -410,6 +411,7 @@ async def begin(self) -> None: try: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -523,7 +525,7 @@ async def _read() -> list[beats.Beat]: verdict = await beats.settle( _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() ) - if verdict.keys_fps != {self.keys_fp}: + if verdict.keys_fps != {self.begun_keys_fp}: raise _refused( "processes", "the root-secret fingerprint changed during attach (a root-secret rotation): " diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index cf83fa8f27..4a820f652b 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -286,7 +286,7 @@ async def _bucket() -> None: try: await step() except Exception as e: # noqa: BLE001 — every nonce is tried; a stale one holds no data - print(f"loom cell prepare: the probe nonce in {where} stays: {e}", file=sys.stderr) + print(f"loom cell: the probe nonce in {where} stays: {e}", file=sys.stderr) def _register_timeout() -> httpx.Timeout: diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..3e88dbafca 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -397,8 +397,8 @@ "options: -h, --help show this help message and exit --name NAME" ], "loom doctor": [ - "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--token TOKEN]", - "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --token TOKEN Override the client's token." + "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--pg-allow-role PG_ALLOW_ROLE] [--stray-orgs] [--stray-orgs-budget STRAY_ORGS_BUDGET] [--token TOKEN]", + "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --pg-allow-role PG_ALLOW_ROLE On a cell: a Postgres role that may make an org held (the servers' role). --stray-orgs On a cell: also read the orgs its stores hold without a tenancy row. --stray-orgs-budget STRAY_ORGS_BUDGET Seconds --stray-orgs may read for, smallest tables first (default 60). --token TOKEN Override the client's token." ], "loom report": [ "usage: loom report [-h] [--open]", @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -459,6 +459,10 @@ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell attach": [ + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..3d597c1e03 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -11213,6 +11213,9 @@ "json", "client", "server_url", + "pg_allow_role", + "stray_orgs", + "stray_orgs_budget", "token" ] } @@ -11280,6 +11283,51 @@ "help": "Override the client's server URL.", "metavar": null }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "On a cell: a Postgres role that may make an org held (the servers' role).", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs" + ], + "dest": "stray_orgs", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "On a cell: also read the orgs its stores hold without a tenancy row.", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs-budget" + ], + "dest": "stray_orgs_budget", + "action": "_StoreAction", + "default": 60.0, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds --stray-orgs may read for, smallest tables first (default 60).", + "metavar": null + }, { "option_strings": [ "--token" @@ -12903,6 +12951,211 @@ "metavar": null } ] + }, + "attach": { + "prog": "loom cell attach", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_attach:cmd_attach" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "wait", + "scan_budget", + "dry_run", + "ch_allow_user", + "ch_allow_user_directory", + "ch_allow_object", + "ch_allow_replica", + "pg_allow_role", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds the waits share", + "metavar": null + }, + { + "option_strings": [ + "--scan-budget" + ], + "dest": "scan_budget", + "action": "_StoreAction", + "default": 7200, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds one full org scan may take", + "metavar": null + }, + { + "option_strings": [ + "--dry-run" + ], + "dest": "dry_run", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run)", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user" + ], + "dest": "ch_allow_user", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user that may write the configured databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user-directory" + ], + "dest": "ch_allow_user_directory", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user directory the check cannot read", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-object" + ], + "dest": "ch_allow_object", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse object (db.name) that writes the databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-replica" + ], + "dest": "ch_allow_replica", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A replica outside the cluster (db.table:replica)", + "metavar": null + }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A Postgres role that may make an org held (the servers' role)", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] } }, "aliases_of": { @@ -12910,7 +13163,8 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", - "prepare": "loom cell prepare" + "prepare": "loom cell prepare", + "attach": "loom cell attach" }, "choices_help": [ { @@ -12937,6 +13191,11 @@ "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", "metavar": "prepare" + }, + { + "dest": "attach", + "help": "Attach the prepared first cell: place its orgs, serve.", + "metavar": "attach" } ] } diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 015cebd948..1c725205e6 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -680,3 +680,20 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: def test_refusal_text() -> None: assert str(Refusal("a", "b")) == "a: b" + + +@pytest.mark.asyncio +async def test_a_root_secret_change_during_attach_refuses_at_the_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.begun_keys_fp = "before" + + async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: + return beats.Verdict(live=1, keys_fps={"after"}) + + monkeypatch.setattr(beats, "settle", settle) + with pytest.raises(Refused, match="a root-secret rotation"): + await run.final_checks(beats.Watch(None, "", "first")) + run.begun_keys_fp = "after" + await run.final_checks(beats.Watch(None, "", "first")) From 738292142572fbd9c971f3274e62ccdc34075748 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:35:26 +0000 Subject: [PATCH 003/181] The widened heartbeat window lives with attach, so server.core imports no boot or insights module instances.read_recent takes the window it reads back; attach's heartbeat module computes it from the live window and a stopping process's drain and retirement budgets. PRUNE_AFTER_S is marked as the contract it is. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach_beats.py | 14 ++++++++++++-- server/core/instances.py | 25 ++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a2fc37701d..d1d938a46b 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -32,6 +32,16 @@ StoppedUnset = Literal["refuse", "wait"] +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return instances.live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + @dataclass(frozen=True) class Beat: """One process: its instance row (`live`: inside the live window and not stopped) and @@ -48,7 +58,7 @@ class Beat: async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: - rows = await instances.read_recent(ch, since=since) + rows = await instances.read_recent(ch, window_s=widened_window_s(), since=since) ids = [str(r["instance_id"]) for r in rows] cells = await ch.query_all( f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " @@ -236,7 +246,7 @@ async def settle( def watch_period_s() -> float: """`T'`: how long a row whose window reaches past the rescan's start is kept before the leader's sweep may drop it, less a margin for clock skew.""" - return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S class Watch: diff --git a/server/core/instances.py b/server/core/instances.py index bb47bfd945..80e3e76896 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -56,7 +56,7 @@ # How old a row's last beat is before the leader drops its key. A contract across builds, # not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader # that sweeps them may run another build, so a change is a design change of its own. -PRUNE_AFTER_S = 3600 +PRUNE_AFTER_S = 3600 # not a setting: a contract across builds # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -134,16 +134,6 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) -def widened_window_s() -> float: - """The live window plus what a stopping process still does after its stop beat: it - drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, - stopped or not, is younger than this may still write.""" - from server.boot import SHUTDOWN_DRAIN_BUDGET_S - from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S - - return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S - - def boot_allowance_s() -> float: """How long a booting process takes to report its first cell row and its first router answer: two beats and one router read.""" @@ -217,10 +207,12 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out -async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: - """Every process of every deployment sharing this ClickHouse whose widened window - (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or - past now without one: stopped rows included, which `read_live` drops. `stopped` is its +async def read_recent( + ch: Any, *, window_s: float, since: str | None = None +) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose last beat plus + `window_s` reaches past `since` (a ClickHouse `DateTime64(3)` as text), or past now + without one: stopped rows included, which `read_live` drops. `stopped` is its newest row's flag and `live` whether that row is inside the live window and not stopped, both judged in ClickHouse.""" rows = await ch.query_all( @@ -238,7 +230,7 @@ async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, An """, params={ "live": int(live_window_s() * 1000), - "wide": int(widened_window_s() * 1000), + "wide": int(window_s * 1000), "since": since or "", }, ) @@ -397,5 +389,4 @@ def _reset_for_tests() -> None: "start", "stop", "tick", - "widened_window_s", ] From 257597973f82ed821f500576b9fd14700eeee9f8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:51:59 +0000 Subject: [PATCH 004/181] ensure_table's integration test lands with attach, its caller Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- .../integration/test_cell_org_triggers_pg.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index e49dde4426..a47f4baec5 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -714,6 +714,32 @@ async def test_missing_names_a_guard_disabled_narrowed_or_calling_another_functi assert "loom.tenancy_refuse_unmarked() has a body or settings no migration writes" in texts +@pytest.mark.asyncio +async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_asked() -> None: + """Attach creates a first cell's triggers in `recording`, one table at a time; a boot's + `ensure` never does, and the function passes every row there.""" + async with _cell("recording") as (dsn, conn): + with pytest.raises(RuntimeError, match="recording"): + await cell_triggers.ensure_table(conn, "scope_generations") + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + assert await _cell_triggers(conn) == { + ("scope_generations", "tenancy_insert", "O"), + ("scope_generations", "tenancy_update", "O"), + ("tenancy", "tenancy_record", "O"), + } + # Again: nothing to do. + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + with pytest.raises(ValueError, match="no cell triggers"): + await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) + plain = await _unmarked(dsn) + try: + await plain.execute(_INSERT, "unheld-org", "s") # recording passes every row + await plain.execute(_HOLD, "new-org") # a first cell still recording records + finally: + await plain.close() + + @pytest.mark.asyncio @pytest.mark.parametrize("loom_cell", ["eu-1", ""]) async def test_the_cell_refresh_reads_the_marker_its_pool_sends( From 79c97d22d48abd072952f595a90067484cd89d72 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 07:48:23 +0000 Subject: [PATCH 005/181] Fix a double space in the failure table's pooler row Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 96580ec7d2..46e1439538 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1292,7 +1292,7 @@ the directory row. | A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | | A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | | A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | | A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | From f2fad614ff5bab7f3dbbf65ab7f416e88c9dcd91 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:10:11 +0000 Subject: [PATCH 006/181] Attach refuses a table its role cannot alter up front, charges --wait in wall time, and says when the mode commit's outcome is unknown The refusals name every table the trigger pass would alter whose owner role attach's role is not a member of, before any write; ensure_table and _run_ddl answer applied, busy or denied, so a denial during the pass refuses at once rather than after --wait as a busy lock. --wait is wall time: each bounded wait charges its elapsed time, lock tries included, and every lock try waits at most what is left; scan time stays outside it. From the mode commit on, a stop that comes before the durable read says the commit's outcome is unknown and to rerun attach, for a refusal and for any other error alike. The prune-window check runs with the refusals, before any write; an enforced resume runs one trigger pass; the lock session opens inside the block that closes it; the durable read retries a timeout or an outage only; refusals name no command that does not exist. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 78 +++--- server/cli/cell_attach_beats.py | 37 ++- server/cli/cell_attach_pg.py | 235 +++++++++++------- server/core/cell_triggers.py | 41 +-- tests/integration/test_cell_attach.py | 35 +++ .../integration/test_cell_org_triggers_pg.py | 7 +- tests/unit/test_cell_attach.py | 124 ++++++++- 7 files changed, 396 insertions(+), 161 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index c8b72fb687..ea9cf77d35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -365,9 +365,9 @@ async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: out += await self.catalog_refusals() out += await self.durability_refusals() rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) - tables_pg = [f"loom.{r['relname']}" for r in rows] - out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await apg.own_role_refusals(self.lock_conn, list(rows)) out += await self.ch_limit_refusals() + out += beats.prune_window_refusals() if out: raise Refused(out) state, _ = await apg.read_mode(self.lock_conn) @@ -449,6 +449,8 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg + # From here on the mode may have risen: only the durable read can say. + self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: @@ -465,22 +467,29 @@ async def commit_mode(self) -> None: async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg_profile + from server.core import pg, pg_profile - while True: - try: + with self.wait.counting(): + while True: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") - read = await apg.durable_read() + try: + read = await apg.durable_read() + except Exception as e: + # A try cut off by its bound, or an outage: its outcome is unknown, and + # the next try reads again. Anything else is no reason to retry. + outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) + if not (isinstance(e, TimeoutError) or outage): + raise + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused( + "postgres durable read", "did not commit within --wait" + ) from e + continue if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break - except Refused: - raise - except Exception as e: - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused("postgres durable read", "did not commit within --wait") from e if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -540,8 +549,6 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.cheap_checks("wait") since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) watch = beats.Watch(self.ch, since, self.name) - if refusals := watch.refusals_up_front(): - raise Refused(refusals) task = asyncio.create_task(watch.run()) try: listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) @@ -567,7 +574,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): raise _refused( "postgres loom.cell_self", - "differs from the directory's fingerprints (a loom cell update between them?)", + "differs from the directory's fingerprints (a fingerprinted setting changed " + "during attach?)", ) state, _ = await apg.read_mode(self.lock_conn) if state != "enforced": @@ -596,7 +604,9 @@ async def done(self) -> bool: if state != "enforced": raise _refused( "postgres cell state", - f"the directory says serving and the cell reads {state}: run loom cell restore", + f"the directory says serving and the cell reads {state}: the directory and the " + "cell's Postgres disagree, as after a restore of one of them; attach lowers " + "and repairs neither", ) return True @@ -606,8 +616,8 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - await self.open_lock() try: + await self.open_lock() await self.read_own_row() topology, names, listed = await self.refusals() if self.args.dry_run: @@ -626,7 +636,6 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.triggers() await self.router.call( "/attach/enforced", {"deployment_id": self.row["deployment_id"]} ) @@ -695,18 +704,30 @@ def log(text: str) -> None: except Refused as e: for r in e.refusals: log(f"refusing: {r}") - mode = ( - "enforced, attaching: rerun attach to resume" - if run.mode == "enforced" - else ("unchanged" if not run.mode else f"{run.mode}") - ) - log(f"the cell stays {mode}") + log(mode_line(run.mode)) + return 1 + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + log(f"stopped ({type(e).__name__}: {e}); run it again") + log(mode_line(run.mode)) return 1 run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code +def mode_line(mode: str) -> str: + """What a stopped run left the cell's mode at, by what it last knew.""" + if mode == "enforced": + return "the cell stays enforced and attaching: rerun attach to resume" + if mode == "unknown": + return ( + "the mode commit's outcome is unknown: rerun attach, which resumes by the mode it reads" + ) + if mode == "recording": + return "the cell stays recording" + return "the cell's state is unchanged" + + def cmd_attach(args: argparse.Namespace) -> int: if problem := _args_problem(args): print(f"loom cell attach: {problem}", file=sys.stderr) @@ -719,14 +740,7 @@ def cmd_attach(args: argparse.Namespace) -> int: from server.core.ch import statement_timeout async def _run(ch: Any) -> int: - try: - return await attach(ch, args, url) - except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes - print( - f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", - file=sys.stderr, - ) - return 1 + return await attach(ch, args, url) # Every ClickHouse statement, the connect included, may run as long as a scan may. with statement_timeout(args.scan_budget): diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index d1d938a46b..a9c6e80377 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -209,6 +209,16 @@ async def settle( """Read and judge until nothing is waited on: Refused at the first refusal, and once `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended once, to `extend_s` past the first sight of the last process first seen booting.""" + with wait.counting(): + return await _settle(read, judge_beats, wait, extend_s) + + +async def _settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + extend_s: float, +) -> Verdict: first_seen: dict[str, float] = {} extended = False while True: @@ -220,7 +230,7 @@ async def settle( raise Refused(refusals) return verdict for i in verdict.booting: - first_seen.setdefault(i, wait.spent) + first_seen.setdefault(i, wait.used()) if await wait.pause(): continue if extend_s and not extended and verdict.booting: @@ -249,6 +259,20 @@ def watch_period_s() -> float: return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S +def prune_window_refusals() -> list[Refusal]: + """A heartbeat interval so long that the leader may sweep a stopped process's rows + before step 4's watch reads them refuses before any write.""" + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + class Watch: """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at @@ -265,17 +289,6 @@ def __init__( self.gap = False self._good = clock() - def refusals_up_front(self) -> list[Refusal]: - if watch_period_s() < 2 * _READ_TIMEOUT_S: - return [ - Refusal( - "LOOM_INSTANCE_HEARTBEAT_SECONDS", - "so long that a stopped process's rows can be swept before attach reads " - "them: lower it", - ) - ] - return [] - async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 6ee4432907..076dc4a196 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -10,8 +10,9 @@ from __future__ import annotations import asyncio +import contextlib import time -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -23,38 +24,59 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval -# The migration lock's wait a trigger try (a boot holds it for its migrations). -_MIGRATION_LOCK_TIMEOUT = "10s" -# The mode commit's lock wait a try, and the statement bound above it. -_MODE_LOCK_TIMEOUT = "2s" -_MODE_STATEMENT_TIMEOUT = "5s" +# The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a +# try never waits longer than --wait has left. +_MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# The mode commit's lock wait a try, in ms, and its statement bound's margin above it. +_MODE_LOCK_MS = 2_000 # not a setting: one try's bound +_MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} class Wait: - """`--wait`: the seconds attach's waits share, counting only the time spent waiting - (never scan time).""" + """`--wait`: the wall-clock seconds attach's bounded waits share. Each wait runs inside + `counting()`, which charges its elapsed time, lock tries included; time outside every + wait (a scan) is never charged.""" def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: self.seconds = float(seconds) self.spent = 0.0 self._clock = clock + self._since: float | None = None + + def used(self) -> float: + """The seconds charged so far, the wait running now included.""" + running = 0.0 if self._since is None else self._clock() - self._since + return self.spent + running def left(self) -> float: - return max(0.0, self.seconds - self.spent) + return max(0.0, self.seconds - self.used()) + + @contextlib.contextmanager + def counting(self) -> Iterator[None]: + """Charge the block's wall time (a block inside another is charged once).""" + if self._since is not None: + yield + return + self._since = self._clock() + try: + yield + finally: + self.spent += self._clock() - self._since + self._since = None - def now(self) -> float: - return self._clock() + def bound_ms(self, most_ms: int) -> int: + """A lock timeout for one try: `most_ms`, or what is left when that is less.""" + return min(most_ms, int(self.left() * 1000)) async def pause(self, seconds: float = POLL_S) -> bool: - """Wait `seconds` (at most what is left) and count them; False once none is left.""" + """Wait `seconds` (at most what is left); False once none is left.""" step = min(seconds, self.left()) if step <= 0: return False - start = self._clock() - await _sleep(step) - self.spent += max(step, self._clock() - start) + with self.counting(): + await _sleep(step) return True @@ -137,7 +159,7 @@ async def take_lock(conn: Any, what: str) -> None: [ Refusal( "attach lock", - f"held by {holder}: an attach, prepare or update is running; {what} " + f"held by {holder}: an attach or a prepare is running; {what} " "runs one at a time", ) ] @@ -364,14 +386,22 @@ async def session_passes() -> bool: """ -async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: - """What attach's own role lacks (the guarded tables' owner roles are `missing`'s - `owner` defects), and a server that allows prepared transactions.""" +async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: + """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` + read: its grants, `SELECT` on every org table, and membership in the owner role of every + table it gives triggers (or the trigger pass could not alter it); and a server that + allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] - for table in tables: - if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): - lacks.append(f"SELECT on {table}") + for r in tables: + if not await conn.fetchval( + "SELECT has_table_privilege($1, 'SELECT')", f"loom.{r['relname']}" + ): + lacks.append(f"SELECT on loom.{r['relname']}") + planned = set(cell_triggers.planned_names(tables)) + for r in tables: + if r["relname"] in planned and not r["owned"]: + lacks.append(f"membership in the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -400,51 +430,66 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield - by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" - from server.core.schema.pg_migrate import _LOCK_KEY - + by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. + Then `missing`: any defect refuses.""" deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " "FROM pg_settings WHERE name = 'deadlock_timeout'" ) ) - out: list[Refusal] = [] - for table in await cell_triggers.planned_tables(conn): - yielded = False - while True: - lock_timeout = cell_triggers.LOCK_TIMEOUT - holders = await cell_triggers.holders(conn, table) - if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_timeout, yielded = f"{deadlock_ms + 500}ms", True - done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) - if done: - break - if not await wait.pause(): - out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) - break - log(f"loom.{table}: its lock is busy; retrying") - if out: - return out + with wait.counting(): + for table in await cell_triggers.planned_tables(conn): + if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] -async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: - await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") +async def _ensure_table( + conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] +) -> Refusal | None: + yielded = False + while True: + lock_ms = 100 + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_ms, yielded = deadlock_ms + 500, True + lock_ms = wait.bound_ms(lock_ms) + result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) + if result == "applied": + return None + if result == "denied": + return Refusal( + f"postgres loom.{table}", + "attach's role cannot alter it: make the role a member of its owner role", + ) + if not await wait.pause(): + return _busy_refusal(table, await cell_triggers.holders(conn, table)) + log(f"loom.{table}: its lock is busy; retrying") + + +async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: + """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) + if migration_ms <= 0: + return "busy" + await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: - await conn.execute(_MIGRATION_LOCK, key) + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return False # a boot migrating: the next try + return "busy" # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) finally: - await conn.execute(_MIGRATION_UNLOCK, key) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -487,49 +532,56 @@ async def commit_enforced( log: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, - within `wait`; `confirm_lock` before each try and each termination (it refuses when - another run holds the attach lock), and `ready` before each try (False skips the try - while a process boots). After a failed try the in-flight mode transaction, found by its - lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the - durable read's to say.""" - while True: - await confirm_lock() - if await ready(): - try: - async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") - await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", f"the mode update read {status}" - ) - ] - ) - return - except Refused: - raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less + when `wait` has less left), within `wait`; `confirm_lock` before each try and each + termination (it refuses when another run holds the attach lock), and `ready` before + each try (False skips the try while a process boots). After a failed try the in-flight + mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; + whether the commit landed is the durable read's to say.""" + with wait.counting(): + while True: + await confirm_lock() + lock_ms = wait.bound_ms(_MODE_LOCK_MS) + if lock_ms > 0 and await ready(): + if await _mode_try(conn, lock_ms): + return log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) - if not await wait.pause(): - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", - "its lock was not taken within --wait: a long transaction reads it " - "FOR SHARE; rerun attach", - ) - ] + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads " + "it FOR SHARE; rerun attach", + ) + ] + ) + + +async def _mode_try(conn: Any, lock_ms: int) -> bool: + """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + return True + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: @@ -626,6 +678,11 @@ async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: """Wait until no backend of the cell's database holds an xid or a snapshot older than `xmax`; Refused at once on a prepared transaction or an unreadable row, and past `wait` naming each backend waited on and each other database's horizon holder.""" + with wait.counting(): + await _wait_transactions(conn, xmax, wait) + + +async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: while True: if refusals := await prepared_refusals(conn): raise Refused(refusals) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 9c2c9755ae..516dee4cc1 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any +from typing import Any, Literal from server.core import cell, pg, readiness @@ -92,6 +92,9 @@ AND l.mode NOT IN ('AccessShareLock', 'RowShareLock') """ +# What a table's DDL came to (`ensure_table`, `_run_ddl`). +Applied = Literal["applied", "busy", "denied"] + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -242,7 +245,12 @@ async def ensure(conn: Any) -> None: async def planned_tables(conn: Any) -> list[str]: """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" - return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + return planned_names(await conn.fetch(ORG_TABLES)) + + +def planned_names(rows: list[Any]) -> list[str]: + """`planned_tables` from rows `ORG_TABLES` read.""" + return [r["relname"] for r, _ in _planned(rows)] async def ensure_table( @@ -252,14 +260,14 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: +) -> Applied: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function - passes every row and a boot never creates them. False when the DDL was not applied (its - lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); - True when it was, or nothing was needed. Raises for a table outside the planned set, or - in another state.""" + passes every row and a boot never creates them. `applied` when the DDL ran or nothing + was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, + `denied` when this role cannot alter the table. Raises for a table outside the planned + set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -270,7 +278,9 @@ async def ensure_table( raise ValueError(f"loom.{table} carries no cell triggers") ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) - return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + if not sqls: + return "applied" + return await _run_ddl(conn, table, sqls, lock_timeout, attempts) def _table_statements( @@ -293,10 +303,11 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The - boot serves past the two causes cells.md names (its lock not taken, a table this role - cannot alter); any other error fails it, as a failed read of the state does.""" +) -> Applied: + """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the + two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: + a table this role cannot alter), which a boot serves past; any other error fails it, as + a failed read of the state does.""" import asyncpg for attempt in range(attempts): @@ -305,12 +316,12 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return True + return "applied" except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return False + return "denied" except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -323,7 +334,7 @@ async def _run_ddl( "; ".join(sqls), await _holders(conn, table), ) - return False + return "busy" async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 406d277646..9bfe3fe9bf 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -633,3 +633,38 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( + cell_stores, +) -> None: + from server.core import cell_triggers + + await _prepare(cell_stores, _Network()) + role = f"it_o_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA loom TO {role}" + ) + await conn.execute(f"SET ROLE {role}") + rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) + refusals = await cell_attach_pg.own_role_refusals(conn, rows) + lacking = [r.reason for r in refusals if "owner role" in r.reason] + assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert any("loom.scope_generations" in r for r in lacking) + # The trigger pass tells the cause apart: denied, not a busy lock. + assert ( + await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + == "denied" + ) + await conn.execute("RESET ROLE") + assert await _mode() == "recording" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index a47f4baec5..b2fb66a467 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -721,15 +721,16 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a async with _cell("recording") as (dsn, conn): with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + ensure = cell_triggers.ensure_table + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert await ensure(conn, "tenancy", recording_first=True) == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 1c725205e6..b57a40eb57 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -222,12 +222,72 @@ async def forever() -> list[beats.Beat]: @pytest.mark.asyncio -async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: - wait = apg.Wait(2, clock) - clock.now += 100 # a scan's time - assert await wait.pause() and await wait.pause() - assert not await wait.pause() - assert wait.spent == 2.0 +async def test_wait_is_wall_time_lock_tries_included_and_no_scan(clock: _Clock) -> None: + """Each mode-commit try waits its lock timeout before it times out: the tries and the + pauses between them end within --wait, and time outside the wait is never charged.""" + asked: list[str] = [] + + def lock(sql: str, *_a: Any) -> str: + if "lock_timeout" in sql: + asked.append(sql) + clock.now += int(sql.split("'")[1].removesuffix("ms")) / 1000 + raise _Timeout() + return "OK" + + conn = FakeConn( + [ + ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + + async def nothing() -> None: + pass + + async def ready() -> bool: + return True + + wait = apg.Wait(4, clock) + clock.now += 100 # a scan before the wait + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + assert clock.now - 100 <= 4 and wait.spent <= 4 + # A 2 s try, a 1 s pause, then a try that waits only the second left. + assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] + + +@pytest.mark.asyncio +async def test_the_trigger_pass_refuses_at_once_a_table_it_cannot_alter( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def ensure_table(*_a: Any, **_kw: Any) -> str: + return "denied" + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", holders) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("deadlock_timeout", 1000)]) + wait = apg.Wait(60, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert refusal.subject == "postgres loom.scope_generations" + assert "cannot alter it" in refusal.reason and wait.spent < 1 + + async def busy(*_a: Any, **_kw: Any) -> str: + clock.now += 0.1 + return "busy" + + monkeypatch.setattr(cell_triggers, "ensure_table", busy) + wait = apg.Wait(3, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert "not taken within --wait" in refusal.reason and wait.spent <= 3 # -- the rescan's watch ---------------------------------------------------------------- @@ -272,9 +332,9 @@ def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) - assert beats.Watch(None, "", "first").refusals_up_front() + assert beats.prune_window_refusals() monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) - assert beats.Watch(None, "", "first").refusals_up_front() == [] + assert beats.prune_window_refusals() == [] # -- batches, ordering, arguments -------------------------------------------------------- @@ -491,10 +551,19 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ("max_prepared_transactions", 5), ] ) - refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + tables = [ + {"relname": "memory_current", "relkind": "r", "owned": True}, + # Its grants are held, but not its owner role's membership: the trigger pass could + # not alter it, so it refuses before any write. + {"relname": "scope_generations", "relkind": "r", "owned": False}, + {"relname": "tenancy", "relkind": "r", "owned": True}, + ] + refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks membership in pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres role attacher: attach's role lacks membership in the owner role of " + "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", ] @@ -674,7 +743,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] assert await run.done() state[0] = "recording" - with pytest.raises(Refused, match="run loom cell restore"): + with pytest.raises(Refused, match="attach lowers and repairs neither"): await run.done() @@ -697,3 +766,38 @@ async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: await run.final_checks(beats.Watch(None, "", "first")) run.begun_keys_fp = "after" await run.final_checks(beats.Watch(None, "", "first")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [Refused([Refusal("postgres", "cut")]), OSError("gone")]) +async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outcome_is_unknown( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: Exception +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def landed(*_a: Any) -> None: + pass # the commit landed; its answer may not have + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def run(self: Any) -> int: + self.row = {"deployment_id": "dep-1"} + await self.commit_mode() + raise failure # what stops the run before the durable read finishes + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "commit_enforced", landed) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "run", run) + url = httpx.URL("http://127.0.0.1:7900") + assert await cell_attach.attach(None, _args(), url) == 1 + err = capsys.readouterr().err + assert "the mode commit's outcome is unknown: rerun attach" in err + assert "stays recording" not in err From dcf7996e290d83efdade60734379ae1c33bee5d4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:17:07 +0000 Subject: [PATCH 007/181] Doctor's stray-org read names a table it cannot read and goes on, and the attach docs describe one step numbering and today's procedure --stray-orgs labels only a spent budget as tables left unchecked; any other refusal names its table and the read continues. A failed catalog read reports both of attach's Postgres checks as not read. The runbook and doctor describe removing a stray or over-long org by hand, and name no command that does not exist yet. cells.md labels the first attach's list with the design's step names, so step 1 to step 4 mean the same everywhere, rewrites the stuck-process failure row and the planned tests for a mode that only rises, and says --wait is wall time. The trigger-cost harness uses cell_triggers.create_statement and deletes the rows its inserts wrote. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 21 +- docs/architecture/cells.md | 405 +++++++++++++------------- docs/operations/cell-attach.md | 19 +- server/cli/cell_attach.py | 41 +-- server/cli/doctor_cell.py | 36 ++- server/core/cell_triggers.py | 5 + tests/integration/test_cell_attach.py | 20 ++ tests/unit/test_cell_attach.py | 27 ++ 8 files changed, 325 insertions(+), 249 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e90fbb32fa..4363a810d6 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,9 +1,10 @@ """The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays). Run it against an attached cell's server, which must not serve anything else, -with a writer token for a placed org; it drops and recreates the cell's org triggers, so -never against a deployment in use. Needs no provider. +guard stays), deleting the rows its inserts wrote at the end. Run it against an attached +cell's server, which must not serve anything else, with a writer token for a placed org; +it drops and recreates the cell's org triggers, so never against a deployment in use. +Needs no provider. PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ --base-url http://127.0.0.1:7932 --pg-dsn \\ @@ -36,7 +37,7 @@ async def _triggers(dsn: str, on: bool) -> None: for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: - await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + await conn.execute(cell_triggers.create_statement(table, name)) enabled = await conn.fetchval( "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", list(cell_triggers.ORG_TRIGGERS), @@ -111,6 +112,18 @@ async def main() -> None: ) finally: await _triggers(args.pg_dsn, True) + await _clean(args) + + +async def _clean(args: argparse.Namespace) -> None: + """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" + conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) + try: + await conn.execute( + "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + ) + finally: + await conn.close() if __name__ == "__main__": diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 46e1439538..3ea7ddab80 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -682,7 +682,8 @@ mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_ this database (a pid captured at connect is wrong behind a connection path that swaps backends), and waits until the lock clears. Whether the commit landed is decided by a durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the +durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -881,14 +882,13 @@ read-only scope, lists any tenancy row the directory does not place. yet, a later cell's configured orgs at attach and a changed one at `loom cell update` included, passes the gate, a closed cell and `placements_max` included, so a later cell closed or full before attach refuses the attach, naming the org. -- **The first cell records until `enforced` as a single deployment does.** In - `recording`, which lasts until step 3 writes `enforced` to the cell's mode (and again - whenever step 3 or step 4 returns the cell to it), a verified request for a new org - writes a tenancy row with no placement call, so the router's rate, `placements_max` - and `loom cell limit --closed` do not apply: new orgs arrive as they do on a single - deployment, bounded by its own signup, issuer and JIT settings, and step 2 or step 4 - places them unrefused. `loom doctor` lists the recorded orgs that hold no data, so the - operator sees what attach will place. +- **The first cell records until `enforced` as a single deployment does.** In `recording`, + which lasts until step 3 writes `enforced` to the cell's mode (the mode only rises after + it), a verified request for a new org writes a tenancy row with no placement call, so the + router's rate, `placements_max` and `loom cell limit --closed` do not apply: new orgs + arrive as they do on a single deployment, bounded by its own signup, issuer and JIT + settings, and step 2 or step 4 places them unrefused. `loom doctor` lists the recorded + orgs that hold no data, so the operator sees what attach will place. - **Configured orgs are placed at attach.** The cell's `oidc_jit_org`, the `self_signup_org` of a cell with signup on, and its code orgs (`LOOM_CODE_ORG` and the org of each `LOOM_CODE_REPOS_JSON` entry the code config keeps) and tracker org are @@ -986,8 +986,8 @@ The first attach (`loom cell attach`, its procedure on [attaching the first cell](../operations/cell-attach.md)) turns the existing deployment into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at `attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an -advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and -`loom cell update` take the same lock), after checking that the session is one direct +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare +takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked writer's. A lock found lost between steps is taken again (refusing when another run holds @@ -1034,74 +1034,76 @@ It refuses, before any write, naming each: `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user reads. -Then, in order: - -1. `attach/begin`, on every run, before any long step: the router probes the stores - attach wrote its nonces to, checks the upstream, and writes the fingerprints and - `keys_fp`; a directory restored to before it is written again. -2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, - through `attach/enforced` again, to the durable read. -3. Create the triggers itself, on every run, before step 2's scan and again just before the - mode commit (so a table a migration added during the scan is guarded from the commit - on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock - and retried a second apart within `--wait`; an autovacuum holding it gets one try just - past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then - `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the - triggers, which pass every row there. -4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads - every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an - `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with - the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not - read: a log row keeps the org a caller typed before any check, so placing a log-only - org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such - orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row - is written; a name no request can carry is skipped and named. Then the tenancy rows of - the orgs found, on a marked session, and no placement: unscoped callers are answered - from the tenancy rows, and placing now would move the point of no return earlier. -5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session - checks and the standbys again, and the scan's elapsed time within half of - `--scan-budget`, since step 4 reads the same tables again. Each refuses in - `recording`, which can still be backed out. -6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on - `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock - (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database - alone) and ended. From the commit on the mode only rises: every later refusal leaves - the cell `enforced` and `attaching`, and a rerun resumes from the durable read. -7. The durable read, in every run that reads `enforced`: the mode and the `first` row read - in a transaction that emits a transactional logical message, whose commit waits for - the synchronous standbys, with the standbys checked attached just before and after. - Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED - writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a - snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is - assigned after the mode commit. -8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the - keys, unmarked, or with no router where it serves, refuses at once. No process depends - on its cached mode (a request records a new org through `tenancy._RECORD`, which reads - the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. -9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling - one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` - (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach - refuses, naming each, and each backend of another database holding the horizon. A - prepared transaction, or a row of the cell's database whose state is hidden, refuses - at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the - heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); - the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery - tables move rows to ClickHouse), and the queue and drain tables and the tables a view - reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, - in `attach/place` batches; the writer check again; and the final checks, under a floor - of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): - the heartbeats, read through the rescan every half of the time the leader keeps a - stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` - unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` - equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. - -`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 -hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would -be placed and skipped and the scan's time, and writes nothing; the router's checks at -`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is -done (exit 0), anything else refuses, naming `loom cell restore`. +Then, in order (the step numbers are the ones the rest of this page uses): + +- **`attach/begin`**, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +- **Step 1, the mode**: `recording`, or `enforced` on a resume, which goes from the + triggers, through `attach/enforced` again, to the durable read. +- **The triggers**: created by attach itself, on every run, before step 2 and again just + before step 3 (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +- **Step 2, the scan**, in `recording`: one statement per table, no long transaction (step 4 + reads every source again): the Postgres tables (`loom.tenancy` among them) by skip scans + on an `org`-led btree index, then the ClickHouse tables, each flushed and synced first, + with the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only org + would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such orgs. + An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row is + written; a name no request can carry is skipped and named. Then the tenancy rows of the + orgs found, on a marked session, and no placement: unscoped callers are answered from the + tenancy rows, and placing now would move the point of no return earlier. +- **The pre-commit checks**: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, + since step 4 reads the same tables again. Each refuses in `recording`, which can still be + backed out. +- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE + lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock (only + attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) + and ended. From the commit on the mode only rises: every later refusal leaves the cell + `enforced` and `attaching`, and a rerun resumes from the durable read. +- **The durable read**, in every run that reads `enforced`: the mode and the `first` row + read in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. Then + `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED writer + that read `recording` holds an xid below `X`, and a REPEATABLE READ one a snapshot older + than `X`: the trigger functions are `VOLATILE`, and the message's xid is assigned after + the mode commit. +- **The heartbeats** again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends on + its cached mode (a request records a new org through `tenancy._RECORD`, which reads the + mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +- **Step 4**: wait, within `--wait`, while any backend of the cell's database but the + polling one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A prepared + transaction, or a row of the cell's database whose state is hidden, refuses at once. Then + the cheap checks (the writer check, `missing`, the Postgres checks, the heartbeats with + every stopped process with `LOOM_CELL` unset aged out, the standbys); the rescan of every + source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to + ClickHouse), and the queue and drain tables and the tables a view reads before the others + (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; + the writer check again; and the final checks, under a floor of their own (the larger of + what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the + rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), + refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to + the directory's fingerprints, one `keys_fp` equal to the one `begin` wrote, the mode + `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try +included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each +full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints +what would be placed and skipped and the scan's time, and writes nothing; the router's +checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: +`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's +Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1297,13 +1299,13 @@ the directory row. | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | | A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | | A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | | Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | | A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | | A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | | A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | -| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until they are removed by hand (the runbook's procedure); `loom doctor --stray-orgs` names it | | An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | | A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | | `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | @@ -1474,126 +1476,117 @@ An integration test with two cells and a router in CI's Postgres job: - every route, the batch entries of MCP and every CLI command included, refuses another cell's org and writes nothing in its Postgres or ClickHouse (the route, op-field and CLI censuses cover the rest); -- the first attach places every org with a row in any table, an org first seen during - the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a - bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` - of failed reads of its own rows or mode is gated and serves again once a read lands; a - pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer - request, passes its health probe, answers the probe route and runs no worker on an - attached cell and on a later cell between prepare and attach, keeps serving on a first - cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included; a CLI command - declaring an org with it unset is refused on an `enforced` or `partial` cell, an - org-free one (a token revocation) runs, also with Postgres down on a deployment never - prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds - with the cell's Postgres down; after `loom cell credential --cell --next` both - credentials place and read state, `--finish` refuses the old credential, for a cell - finished before its pods hold the new one the router refuses its calls with 401, its - request answers 503 writing nothing, and it recovers by recording the old hash again, - `--revert` drops a pending `next`, which is then refused, and the old credential keeps - placing, and a second `--next` while one is pending and a hash another cell records as - current or next are refused; `loom cell limit` and `loom cell credential --next`, - `--finish` and `--revert` refuse while the directory is unavailable; concurrent - placements at one under `placements_max` across two router replicas write one; a CLI - command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a - first cell once it is `enforced` (`attaching` included) and on a later cell, and on a - first cell in `recording` runs and records the org; attach with `placements_max` below - the orgs the first cell holds places them all, and a later cell's attach past its - quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step - 2 and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's - mode, a process whose cache still reads `recording` sends a new org's request to the - placement call, which a closed cell answers 503; a CLI command whose insert finds - `enforced` refuses, naming the org, and makes no placement call; after step 3's - refused wait or step 4's recheck returns the cell to `recording`, a new org is - recorded with no placement call, and one at a process whose cache still reads - `enforced` places through the gate until the cache expires; each ordering of attach's - write of the mode with a recording insert holds as stated above (running, starting - during the write, a stream of overlapping inserts each shorter than a try, with attach - setting `enforced` within a try or two, held across several tries, held until too - little of `--wait` is left, a stalled attach, a commit cut off by its timeout, a - commit whose connection is lost while it waits for the standby, read back only once - that backend has gone and written again durably if it reads `enforced`); a - `REPEATABLE READ` transaction whose first statement ran before `enforced` committed, - on attach's role or a second role, holds step 4 until it ends and its write of an - unheld org is then placed, one whose first statement ran after is refused by the - trigger, a transaction that starts after attach takes the lock but before its commit - is visible holds step 4, a transaction opened in `recording` that outlives one run's - `--wait` holds the rerun's step 4, after a refusal and after `loom cell restore` - raised the mode, an autovacuum worker on a loom table does not hold step 4 and a - background worker with a transaction open since `recording` does, a prepared - transaction (after a restart with `max_prepared_transactions` raised during the wait) - makes step 4 refuse at once, naming it, with the mode still `enforced` and a rerun - refused until it is resolved and the setting is back to 0, a write transaction open in - a second database on the same server holds step 4 and is named, a long read on the - synchronous standby, or on a second standby, with `hot_standby_feedback` on does not - hold it, nor does a base backup running across it, a write held open on a +- the first attach places every org with a row in any table, an org first seen during the + attach is placed in step 4, and so is one a direct Postgres insert wrote between step 2's + scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object + catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of + its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` + unset is unready, naming the setting, answers 503 to a bearer request, passes its health + probe, answers the probe route and runs no worker on an attached cell and on a later cell + between prepare and attach, keeps serving on a first cell in `recording`, and keeps the + last mode it read through a failed read, `recording` included; a CLI command declaring an + org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token + revocation) runs, also with Postgres down on a deployment never prepared, and a `loom +cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres + down; after `loom cell credential --cell --next` both credentials place and read + state, `--finish` refuses the old credential, for a cell finished before its pods hold the + new one the router refuses its calls with 401, its request answers 503 writing nothing, + and it recovers by recording the old hash again, `--revert` drops a pending `next`, which + is then refused, and the old credential keeps placing, and a second `--next` while one is + pending and a hash another cell records as current or next are refused; `loom cell limit` + and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is + unavailable; concurrent placements at one under `placements_max` across two router + replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set + refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a + later cell, and on a first cell in `recording` runs and records the org; attach with + `placements_max` below the orgs the first cell holds places them all, and a later cell's + attach past its quota or closed refuses, naming the org; a first cell with `--closed` and + `placements_max` set before step 2 attaches, and an org a request records between step 2 + and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a + process whose cache still reads `recording` sends a new org's request to the placement + call, which a closed cell answers 503; a CLI command whose insert finds `enforced` + refuses, naming the org, and makes no placement call; a refusal after step 3's write + leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; + each ordering of attach's write of the mode with a recording insert holds as stated above + (running, starting during the write, a stream of overlapping inserts each shorter than a + try, with attach setting `enforced` within a try or two, held across several tries, held + past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut + off by its timeout, and a commit whose connection is lost while it waits for the standby, + which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran + before `enforced` committed, on attach's role or a second role, holds step 4 until it ends + and its write of an unheld org is then placed, one whose first statement ran after is + refused by the trigger, a transaction that starts after attach takes the lock but before + its commit is visible holds step 4, a transaction opened in `recording` that outlives one + run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom + table does not hold step 4 and a background worker with a transaction open since + `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` + raised during the wait) makes step 4 refuse at once, naming it, with the mode still + `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write + transaction open in a second database on the same server holds step 4 and is named, a long + read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on + does not hold it, nor does a base backup running across it, a write held open on a `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a - long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database - past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the - first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` - above 0 refuses before step 1 with the mode still `recording`, while a later cell's - attach does not require either; a missing or disabled org trigger is refused by attach - and named by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode - row was deleted is refused and the state named by `loom doctor`; on that cell a pod - with `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an - org with it unset is refused, as a pod is; updating, deleting or truncating the - `first` row on an `enforced` cell is refused, also under - `session_replication_role = replica`, and a missing or disabled `first`-row trigger is - refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set - on a cell whose mode row is removed answers 503 to a bearer request and runs no worker - pass, and serves again after a rerun of `loom cell prepare` writes the row back while - it runs; a cell the directory registers with no state, tenancy or own row is named by - `loom doctor`, its processes with `LOOM_CELL` set stay unready, a process with it - unset serves as a single deployment, and `loom cell restore` writes both rows back, an - attached cell then serving again and an unattached one after a prepare rerun; a cell - with both rows absent beside a tenancy row, and one beside only its own row, has the - function refuse an unheld org (a boot creates no trigger there), every process gated - and `loom doctor` naming it, and a prepare rerun writes the rows back; a Postgres-only - restore of an attached first cell to before its prepare is named by `loom doctor` and - closed by `loom cell restore`; a prepare rerun during a pending `loom cell update` on - an attached cell, with processes on both values and `loom.cell_self` lost, writes both - values back and keeps both sets of processes serving, and one before attach refuses - the two configurations; a prepare rerun on an attached cell whose `current` is missing - and whose pods' configuration differs from the directory row refuses, and one before - attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a - credential hash that is neither the row's `current` nor its `next` is refused; a - prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode - row back; a cell with a mode row and no `first` row has the function refuse an unheld - org (a boot creates no trigger there), every process gated and `loom doctor` naming - it, and a prepare rerun writes the row back; a later cell whose mode row is set to - `recording` records nothing, its trigger refuses an unheld org, a process with - `LOOM_CELL` set there serves as in `enforced`, a pod with `LOOM_CELL` unset there is - unready and runs no worker, `loom cell prepare` names the row, a CLI command declaring - an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses - its state read; a build N-1 process's `memory.set` and delivery pass on a - never-prepared deployment after N's migration succeed and write no tenancy row, and on - the first cell in `recording` a boot creates no org trigger, and one created by hand - reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), - an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused - and one that hits an existing row of it is not, a marked session's every write passes, - and the triggers on an `enforced` cell cover every table erasure classifies with an - `org` column but the three exclusions; and an org-keyed write while attach holds the - mode table's lock returns at once, with new-org requests answering within their - deadline and mode-cache reads returning at once throughout; `loom cell restore` after - a refusal at step 3's lock leaves the cell `enforced`, and the resumed attach runs - from step 3's wait; a `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at - boot and named by `loom doctor`; `loom cell credential new` prints a credential of - that form and its SHA-256, with the directory and the stores unreachable; prepare and - `--next` refuse an argument that is not 64 lowercase hex characters, the credential - and its bare 128-character secret included, storing nothing; orgs a first cell records - in `recording` with no data are listed by `loom doctor`; a placement flood with no - quota set is held to the rate limit at each router replica, `loom doctor` names the - cell as one with no quota, and a call refused past it at one replica answers 503 with - a `Retry-After` of at most 60 s, after which its retry at that replica, with no other - call of the cell admitted between, is admitted; a cell's first prepare, with no - directory row yet, passes the deployment-id check; a pod or CLI that lost - `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the authority check; - the first cell's existing tokens, refresh tokens and OAuth clients keep working - through the cutover, a logout through the shared host revokes its refresh token; + transaction on one, opened in `recording` and writing nothing before `enforced`, a long + logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past + `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first + cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 + refuses before step 1 with the mode still `recording`, while a later cell's attach does + not require either; a missing or disabled org trigger is refused by attach and named by + `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted + is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset + is unready and runs no worker, and a CLI command declaring an org with it unset is + refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` + cell is refused, also under `session_replication_role = replica`, and a missing or + disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer + request and runs no worker pass, and serves again after a rerun of `loom cell prepare` + writes the row back while it runs; a cell the directory registers with no state, tenancy + or own row is named by `loom doctor`, its processes with `LOOM_CELL` set stay unready, a + process with it unset serves as a single deployment, and `loom cell restore` writes both + rows back, an attached cell then serving again and an unattached one after a prepare + rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, + has the function refuse an unheld org (a boot creates no trigger there), every process + gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a + Postgres-only restore of an attached first cell to before its prepare is named by `loom +doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell +update` on an attached cell, with processes on both values and `loom.cell_self` lost, + writes both values back and keeps both sets of processes serving, and one before attach + refuses the two configurations; a prepare rerun on an attached cell whose `current` is + missing and whose pods' configuration differs from the directory row refuses, and one + before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given + a credential hash that is neither the row's `current` nor its `next` is refused; a prepare + rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a + cell with a mode row and no `first` row has the function refuse an unheld org (a boot + creates no trigger there), every process gated and `loom doctor` naming it, and a prepare + rerun writes the row back; a later cell whose mode row is set to `recording` records + nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom +cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` + set refuses, naming it, and the router refuses its state read; a build N-1 process's + `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed + and write no tenancy row, and on the first cell in `recording` a boot creates no org + trigger, and one created by hand reads the state and looks nothing up, a write under + `session_replication_role = replica` for an unheld org is skipped (origin triggers), an + `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one + that hits an existing row of it is not, a marked session's every write passes, and the + triggers on an `enforced` cell cover every table erasure classifies with an `org` column + but the three exclusions; and an org-keyed write while attach holds the mode table's lock + returns at once, with new-org requests answering within their deadline and mode-cache + reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` + with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom +doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with + the directory and the stores unreachable; prepare and `--next` refuse an argument that is + not 64 lowercase hex characters, the credential and its bare 128-character secret + included, storing nothing; orgs a first cell records in `recording` with no data are + listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at + each router replica, `loom doctor` names the cell as one with no quota, and a call refused + past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its + retry at that replica, with no other call of the cell admitted between, is admitted; a + cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or + CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the + authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep + working through the cutover, a logout through the shared host revokes its refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the @@ -1607,12 +1600,12 @@ An integration test with two cells and a router in CI's Postgres job: application ends the person there, a platform-admin bot still reaches a cell whose SCIM deactivated everyone, and attach refuses a cell with unprovisioned live subjects or sharing another's OIDC client; -- an attach stopped in step 3 after it set `enforced`, before the wait passed, resumes - with the wait before the SCIM precondition, and one refused by the SCIM precondition - resumes from the wait too, the mode still `enforced`; a later attach refused before - its directory transaction leaves `attach_started_at` empty, so a rerun checks a new - cell's refusals again, and a resumed first attach finds its configured orgs placed in - this cell and carries on; +- an attach stopped after step 3's commit landed, before the durable read, says the commit's + outcome is unknown, and its rerun resumes by the mode it reads; one refused after the + durable read resumes from it, the mode still `enforced`; a later attach refused before its + directory transaction leaves `attach_started_at` empty, so a rerun checks a new cell's + refusals again, and a resumed first attach finds its configured orgs placed in this cell + and carries on; - a cell restored to before the first attach's `enforced` gets its mode back from `loom cell restore`; one whose `loom cell restore` was skipped answers 503 for an org it does not hold, another cell's org included, writes no tenancy row for it, and is diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 12a0cbe247..c51541d06d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -105,17 +105,16 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse always goes forward. From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an -ordinary mint scoped to its org places the org on its first request. A direct call of the -router's operator place route, or a live placement with the cell's credential, cannot read -the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's -credential is the servers' alone. +routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places +the org on its first request. A direct call of the router's operator place route, or a live +placement with the cell's credential, cannot read the cell's Postgres and is accepted once +the directory mode is `enforced`; the cell's credential is the servers' alone. ## Orgs attach does not place - **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is - written. One first found in step 4 is skipped and named. Remove such an org by hand - until the stray-org erase command exists, in this order: its bucket objects, from the + written. One first found in step 4 is skipped and named. Remove such an org by hand, in + this order: its bucket objects, from the keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no memory content is left outside erasure; then its rows in every table with an `org` column of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables @@ -128,9 +127,9 @@ credential is the servers' alone. erasure there. `loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, -smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left -unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available -yet. +smallest tables first within `--stray-orgs-budget` (default 60 s), every table the budget +left unchecked and every table it could not read. Remove a stray org's rows by the same +procedure as an over-long org's. ## After attach diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ea9cf77d35..52abea19c6 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -3,26 +3,27 @@ docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and -goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: - -1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; -2. the refusals, before any write: the stores are the registered ones, the session is - direct and marked, the cell's processes (their heartbeats), the roles, settings and - subscriptions that would let an unchecked writer through, the ClickHouse writers, - the table list, durability, attach's own grants; -3. `attach/begin` (the router probes the stores and writes the fingerprints); -4. the mode: `recording`, or `enforced` on a resume; -5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; -6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the - tenancy rows of the orgs found; the triggers again; the pre-commit checks; -7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode - only rises, so every later refusal leaves the cell `enforced` and `attaching`; -8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction - that could have read `recording`, the rescan, the tenancy rows and their placements - (`attach/place`), the final checks and `attach/serving`. - -`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals -and the scan and prints what would be placed, writing nothing. +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. +Steps 1 to 4 are the design's, as cells.md numbers them: + +- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, the + table list, durability, attach's own grants; +- `attach/begin` (the router probes the stores and writes the fingerprints); +- step 1, the mode: `recording`, or `enforced` on a resume; +- the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +- in `recording`: step 2, the scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; step 3, + `attach/enforced` and the mode commit, from which on the mode only rises, so every + later refusal leaves the cell `enforced` and `attaching`; +- the durable read of `enforced`, the heartbeats, and step 4: the wait for every + transaction that could have read `recording`, the rescan, the tenancy rows and their + placements (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits in wall time, `--scan-budget` each full scan. `--dry-run` runs +the refusals and the scan and prints what would be placed, writing nothing. """ from __future__ import annotations diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index c79723bbbc..9edb8e275d 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -186,7 +186,10 @@ async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[st try: reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run - reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + reads = [ + (name, WARN, f"not read: {_reason(e)}") + for name in ("cell_tenancy_privilege", "cell_unchecked_writers") + ] return _grades(state, marked, marks_all, defects) + reads @@ -264,6 +267,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: budget = cell_scan.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] + refused: list[str] = [] for i, (_n, store, table) in enumerate(work): found = cell_scan.ScanResult() name = table if store == "postgres" else table.canonical @@ -274,9 +278,15 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: else: for raw in await cell_scan.ch_orgs(ch, table, topology, budget): found.add(name, raw) - except Refused: - unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] - break + except Refused as e: + # The budget spent leaves this table and the larger ones unread; any other + # refusal (a table with no org-led index, a replica that cannot sync) names + # this table alone, and the read goes on. + if budget.left() <= 1: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + refused += [str(r) for r in e.refusals] + continue orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) bad = [m for ms in found.malformed.values() for m in ms] if orgs or bad: @@ -290,13 +300,20 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: or "every org found has a tenancy row", ) ] - if unchecked or problems: + if unchecked: out.append( ( "cell_stray_orgs_unchecked", WARN, - f"not read within --stray-orgs-budget ({budget_s:g} s): " - + ", ".join(unchecked + [str(p) for p in problems]), + f"not read within --stray-orgs-budget ({budget_s:g} s): " + ", ".join(unchecked), + ) + ) + if refused or problems: + out.append( + ( + "cell_stray_orgs_unread", + WARN, + "not read: " + "; ".join(refused + [str(p) for p in problems]), ) ) return out @@ -305,8 +322,9 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row (log-only orgs, names no request can carry, rows an unchecked writer left), smallest - tables first within one budget, naming every table left unchecked. Never a FAIL: the - stray-org erase command removes what it names.""" + tables first within one budget, naming every table left unchecked and every table it + could not read. Never a FAIL: the operator removes what it names by hand + (docs/operations/cell-attach.md).""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client, statement_timeout diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 516dee4cc1..4395138b4d 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -176,6 +176,11 @@ def _function_sql(function: str) -> re.Pattern[str]: ) +def create_statement(table: str, name: str) -> str: + """The CREATE TRIGGER for `name` on `loom.`, as `ensure` writes it.""" + return _create(f"loom.{_ident(table)}", name) + + def _create(qualified: str, name: str) -> str: """The CREATE TRIGGER for `name` on the table `qualified` names.""" if name == TENANCY_TRIGGER: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 9bfe3fe9bf..e0f96f409f 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -668,3 +668,23 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + # An org table with no org-led index: its skip scan refuses, the read goes on. + await pg.execute("CREATE TABLE loom.it_no_org_index (id int PRIMARY KEY, org text)") + await pg.execute("INSERT INTO loom.it_no_org_index VALUES (1, $1)", tag + "unindexed") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + assert tag + "stray" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unread"] + assert grade == "WARN" and "postgres loom.it_no_org_index" in detail + assert "cell_stray_orgs_unchecked" not in lines diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b57a40eb57..857d12dfe0 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -801,3 +801,30 @@ async def run(self: Any) -> int: err = capsys.readouterr().err assert "the mode commit's outcome is unknown: rerun attach" in err assert "stays recording" not in err + + +@pytest.mark.asyncio +async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.cli import doctor_cell + + async def before(_ch: Any) -> str: + return "" + + async def marked() -> tuple[str, bool, list[Any]]: + return "enforced", True, [] + + async def unmarked() -> bool: + return False + + async def unread(_roles: Any) -> Any: + raise OSError("the catalog read failed") + + monkeypatch.setattr(doctor_cell, "_deployment_check", before) + monkeypatch.setattr(doctor_cell, "_marked_session", marked) + monkeypatch.setattr(doctor_cell, "_unmarked_session", unmarked) + monkeypatch.setattr(doctor_cell, "_writer_reads", unread) + lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} + for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): + assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") From 6229357309419724937049dbc62c69d768925439 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 008/181] Attach's closing line names the mode it last read, every DDL try ends inside --wait, and doctor warns on the role it runs as The mode is taken from the first state read, set to unknown only once the router accepted attach/enforced, and set from what the durable read saw; an interrupt prints the mode line and goes on. The trigger DDL's lock bound is taken after the migration lock, from what --wait has left then. An ended mode transaction whose lock does not clear refuses, naming its backend. A busy table's holders come back with ensure_table's answer, and attach says them once per change rather than a boot's warning a try. Attach counts the role it runs as among its own; doctor does not. The dead SCIM recheck and the unused scim_ok parameter are gone. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 18 +- server/cli/cell_attach_beats.py | 4 +- server/cli/cell_attach_pg.py | 71 ++++-- server/cli/doctor_cell.py | 2 +- server/core/cell_triggers.py | 55 +++-- tests/integration/test_cell_attach.py | 71 +++++- .../integration/test_cell_org_triggers_pg.py | 6 +- tests/unit/test_cell_attach.py | 208 +++++++++++++++++- 8 files changed, 368 insertions(+), 67 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 52abea19c6..23449ccff9 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -282,6 +282,8 @@ async def stores_refusals(self) -> list[Refusal]: if recorded is None or str(recorded["deployment_id"]) != expected: out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) state, first = await apg.read_mode(self.lock_conn) + if state in cell.MODES: + self.mode = state # what a refusal from here on leaves the cell at if first != (True, self.name): out.append( Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") @@ -353,7 +355,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: async def catalog_refusals(self) -> list[Refusal]: """The Postgres checks that run again before the mode commit and in step 4.""" - holders = await apg.privilege_holders(self.lock_conn) + holders = await apg.privilege_holders(self.lock_conn, exempt_self=True) self.report["own_roles"] = holders.own out = apg.privilege_refusals(holders, self.args.pg_allow_role) out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) @@ -450,9 +452,9 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg - # From here on the mode may have risen: only the durable read can say. - self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + # From the mode transaction on the mode may have risen: only the durable read can say. + self.mode = "unknown" try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( @@ -491,6 +493,8 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break + if read.mode in cell.MODES: + self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -666,9 +670,7 @@ async def precommit(self, found: cell_scan.ScanResult) -> None: ) if out: raise Refused(out) - verdict = await self.check_beats("refuse") - if refusals := beats.final_refusals(verdict): - raise Refused(refusals) + await self.check_beats("refuse") async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: found = await self.scan(topology, listed) @@ -711,6 +713,10 @@ def log(text: str) -> None: log(f"stopped ({type(e).__name__}: {e}); run it again") log(mode_line(run.mode)) return 1 + except BaseException: + # An interrupt: the mode line, then the interrupt as it came. + log(mode_line(run.mode)) + raise run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a9c6e80377..c2a8ed2428 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -180,7 +180,7 @@ def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> N ) -def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: +def final_refusals(verdict: Verdict) -> list[Refusal]: out = [] if not verdict.live and not verdict.booting: out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) @@ -192,7 +192,7 @@ def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: "finish the root-secret rollout first", ) ) - if verdict.scim and not scim_ok: + if verdict.scim: out.append( Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 076dc4a196..9c6e4708d6 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -257,15 +257,16 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: ORDER BY 1, 2 """ -# The roles attach counts as its own: superusers, and a login role that itself owns +# The roles attach counts as its own: superusers, a login role that itself owns # loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the -# triggers, or lower the mode, whatever else it holds). +# triggers, or lower the mode, whatever else it holds), and, for attach only, the role it +# runs as ($2): the operator runs attach, while doctor may run as a CI or monitoring role. _OWN_ROLES = """ SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) -)) OR rolname = current_user +)) OR ($2 AND rolname = current_user) ORDER BY 1 """ @@ -279,9 +280,11 @@ class Holders: others: dict[str, list[str]] -async def privilege_holders(conn: Any) -> Holders: +async def privilege_holders(conn: Any, *, exempt_self: bool) -> Holders: + """The holders of a right `_RIGHTS` counts; `exempt_self` counts the role this session + runs as among the own roles (attach's, not doctor's).""" guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded, exempt_self)] others: dict[str, list[str]] = {} for r in await conn.fetch(_RIGHTS): if r["role"] not in own: @@ -449,42 +452,49 @@ async def _ensure_table( conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: yielded = False + said = "" while True: - lock_ms = 100 + most_ms = 100 holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_ms, yielded = deadlock_ms + 500, True - lock_ms = wait.bound_ms(lock_ms) - result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) - if result == "applied": + most_ms, yielded = deadlock_ms + 500, True + outcome = await _ensure_one(conn, table, most_ms, wait) + if outcome.answer == "applied": return None - if result == "denied": + if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): return _busy_refusal(table, await cell_triggers.holders(conn, table)) - log(f"loom.{table}: its lock is busy; retrying") + # Said again only when the holders change: a retry a second is otherwise silent. + if outcome.holders != said: + said = outcome.holders + log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") -async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: - """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" +async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: + """One try under the migration lock: the migration lock's wait, and then the DDL's lock + wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" from server.core.schema.pg_migrate import _LOCK_KEY migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) if migration_ms <= 0: - return "busy" + return cell_triggers.Outcome("busy") await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return "busy" # a boot migrating: the next try + return cell_triggers.Outcome("busy") # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: + lock_ms = wait.bound_ms(most_ms) + if lock_ms <= 0: + return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) @@ -522,6 +532,9 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: """ _TIMED_OUT = frozenset({"55P03", "57014"}) +# How long attach waits for an ended mode transaction's lock to clear. +_END_ROUNDS = 30 # not a setting: checks of the lock +_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -585,10 +598,13 @@ async def _mode_try(conn: Any, lock_ms: int) -> bool: async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait until its lock - clears; Refused when its role is a superuser and attach's is not.""" - for _ in range(30): - rows = await conn.fetch(_MODE_HOLDERS) + """End an earlier run's mode transaction still in flight and wait, for up to + `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role + is a superuser and attach's is not, or when a backend still holds the lock after the + last check, naming it.""" + rows: list[Any] = [] + for _ in range(_END_ROUNDS): + rows = list(await conn.fetch(_MODE_HOLDERS)) if not rows: return superuser = bool( @@ -608,7 +624,20 @@ async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> N ) await confirm_lock() await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(0.1) + await _sleep(_END_PAUSE_S) + if not (rows := list(await conn.fetch(_MODE_HOLDERS))): + return + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"still holds loom.tenancy_mode's lock after it was ended " + f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " + "once it has gone", + ) + for r in rows + ] + ) # -- the durable read ------------------------------------------------------------------- diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 9edb8e275d..e8a5fc83c1 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -95,7 +95,7 @@ async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, from server.core import pg async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=False) unchecked = await cell_attach_pg.unmarked_refusals(conn, False) roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) return [ diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 4395138b4d..7a1ee5e5b7 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any, Literal +from typing import Any, Literal, NamedTuple from server.core import cell, pg, readiness @@ -95,6 +95,14 @@ # What a table's DDL came to (`ensure_table`, `_run_ddl`). Applied = Literal["applied", "busy", "denied"] + +class Outcome(NamedTuple): + """A table's DDL's answer, and with `busy` the sessions it waited behind.""" + + answer: Applied + holders: str = "" + + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -265,14 +273,15 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: +) -> Outcome: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function passes every row and a boot never creates them. `applied` when the DDL ran or nothing was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, - `denied` when this role cannot alter the table. Raises for a table outside the planned - set, or in another state.""" + `denied` when this role cannot alter the table; a `busy` answer carries the sessions it + waited behind and logs nothing, so the caller says it as often as it retries. Raises + for a table outside the planned set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -284,8 +293,8 @@ async def ensure_table( ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) if not sqls: - return "applied" - return await _run_ddl(conn, table, sqls, lock_timeout, attempts) + return Outcome("applied") + return await _run_ddl(conn, table, sqls, lock_timeout, attempts, warn=False) def _table_statements( @@ -308,11 +317,14 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: + *, + warn: bool = True, +) -> Outcome: """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the - two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: - a table this role cannot alter), which a boot serves past; any other error fails it, as - a failed read of the state does.""" + two causes cells.md names kept it from applying (`busy`: its lock not taken, with the + sessions it waited behind; `denied`: a table this role cannot alter), which a boot + serves past; any other error fails it, as a failed read of the state does. `warn` logs + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" import asyncpg for attempt in range(attempts): @@ -321,25 +333,28 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return "applied" + return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return "denied" + return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue - log.warning( - "cell triggers: not applied on loom.%s (%s): %s; %s", - table, - e, - "; ".join(sqls), - await _holders(conn, table), - ) - return "busy" + held = await _holders(conn, table) + if warn: + log.warning( + "cell triggers: not applied on loom.%s (%s): %s; %s", + table, + e, + "; ".join(sqls), + held, + ) + return Outcome("busy", held) + return Outcome("busy") async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index e0f96f409f..d77592d1dc 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,7 +532,7 @@ async def test_each_right_that_makes_an_org_held_is_named( await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) for sql in grant.format(r=role, owner=owner).split("; "): await conn.execute(sql) - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) assert owner in holders.own assert any(why in w for w in holders.others.get(role, [])), holders.others finally: @@ -550,7 +550,7 @@ async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> N try: async with pg.dedicated() as conn: await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] finally: @@ -631,8 +631,40 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} grade, detail = lines["cell_stray_orgs"] assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail - tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} - assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( + cell_stores, monkeypatch +) -> None: + from server.cli import cell_scan, doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("CREATE TABLE loom.it_small (org text PRIMARY KEY)") + await pg.execute("CREATE TABLE loom.it_large (org text PRIMARY KEY)") + await pg.execute("INSERT INTO loom.it_small VALUES ($1)", tag + "small") + await pg.execute( + "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" + ) + await pg.execute("ANALYZE loom.it_small, loom.it_large") + # The budget runs out on the largest table, as it does on a real cell when that table's + # read is what is too slow: every smaller one was read first. + real, read = cell_scan.pg_orgs, [] + + async def pg_orgs(table: str, budget: Any) -> list[str]: + read.append(table) + if table == "loom.it_large": + await asyncio.sleep(budget.left() + 0.1) + return await real(table, budget) + + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} + assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert tag + "small" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unchecked"] + assert grade == "WARN" and detail.endswith(": loom.it_large") @pytest.mark.asyncio @@ -657,10 +689,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. - assert ( - await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - == "denied" + outcome = await cell_triggers.ensure_table( + conn, "scope_generations", recording_first=True ) + assert outcome.answer == "denied" await conn.execute("RESET ROLE") assert await _mode() == "recording" finally: @@ -688,3 +720,28 @@ async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_store grade, detail = lines["cell_stray_orgs_unread"] assert grade == "WARN" and "postgres loom.it_no_org_index" in detail assert "cell_stray_orgs_unchecked" not in lines + + +@pytest.mark.asyncio +async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_ci_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA loom TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"SET ROLE {role}") + as_attach = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + as_doctor = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + await conn.execute("RESET ROLE") + # Attach runs as the operator's role; a CI role doctor runs as is warned about. + assert role in as_attach.own and role not in as_attach.others + assert role not in as_doctor.own + assert "may insert into loom.tenancy" in as_doctor.others[role] + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index b2fb66a467..c219d05777 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -722,15 +722,15 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") ensure = cell_triggers.ensure_table - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" - assert await ensure(conn, "tenancy", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + assert (await ensure(conn, "tenancy", recording_first=True)).answer == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 857d12dfe0..b3db51ee02 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -268,8 +268,8 @@ async def planned(_c: Any) -> list[str]: async def holders(_c: Any, _t: str) -> list[Any]: return [] - async def ensure_table(*_a: Any, **_kw: Any) -> str: - return "denied" + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("denied") monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", holders) @@ -280,14 +280,22 @@ async def ensure_table(*_a: Any, **_kw: Any) -> str: assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 - async def busy(*_a: Any, **_kw: Any) -> str: + holders = iter(["pid 7", "pid 7", "pid 7", "pid 8"] + ["pid 8"] * 100) + + async def busy(*_a: Any, **_kw: Any) -> Any: clock.now += 0.1 - return "busy" + return cell_triggers.Outcome("busy", next(holders)) monkeypatch.setattr(cell_triggers, "ensure_table", busy) - wait = apg.Wait(3, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) - assert "not taken within --wait" in refusal.reason and wait.spent <= 3 + wait = apg.Wait(10, clock) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # Said once per change of holders, not once a try. + assert logged == [ + "loom.scope_generations: its lock is busy (pid 7); retrying", + "loom.scope_generations: its lock is busy (pid 8); retrying", + ] # -- the rescan's watch ---------------------------------------------------------------- @@ -828,3 +836,189 @@ async def unread(_roles: Any) -> Any: lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") + + +@pytest.mark.asyncio +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: + """A migration lock that took the rest of --wait leaves no DDL try.""" + + def migration_lock(*_a: Any) -> str: + clock.now += 2.0 # the rest of --wait + return "OK" + + conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) + wait = apg.Wait(2, clock) + with wait.counting(): + outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) + assert outcome.answer == "busy" + assert not any( + "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + ) + + +@pytest.mark.asyncio +async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: + held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] + conn = FakeConn( + [ + ("l.mode = 'ExclusiveLock'", held), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused) as e: + await apg.end_mode_transactions(conn, confirm) + (refusal,) = e.value.refusals + assert refusal.subject == "postgres backend 5" + assert "still holds loom.tenancy_mode's lock" in refusal.reason + + +def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Run, "run", body) + + +@pytest.mark.asyncio +async def test_a_router_refusal_of_attach_enforced_leaves_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: + raise Refused([Refusal("router", f"refused {path}")]) + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + await self.commit_mode() + return 0 + + monkeypatch.setattr(cell_attach.Router, "call", refuse) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_a_durable_read_that_sees_recording_says_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + async def read() -> apg.Durable: + return apg.Durable(mode="recording", first=(True, "first"), xid=1) + + async def run(self: Any) -> int: + self.mode = "unknown" # a mode commit whose outcome was lost + await self.durable() + return 0 + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + monkeypatch.setattr(apg, "durable_read", read) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_an_enforced_resume_refused_early_says_enforced( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def recorded(_ch: Any) -> dict[str, Any]: + return {"deployment_id": "other"} + + async def health(_self: Any, _host: str) -> str: + return "dep-1" + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return "enforced", (True, "first") + + async def run(self: Any) -> int: + self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} + self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) + if refusals := await self.stores_refusals(): + raise Refused(refusals) + return 0 + + monkeypatch.setattr(pg_profile, "recorded_authority", recorded) + monkeypatch.setattr(cell_attach.Router, "health", health) + monkeypatch.setattr(apg, "read_mode", read_mode) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + err = capsys.readouterr().err.splitlines() + assert "clickhouse: records another deployment id" in err[0] + assert err[-1] == ( + "loom cell attach: the cell stays enforced and attaching: rerun attach to resume" + ) + + +@pytest.mark.asyncio +async def test_an_interrupt_prints_the_mode_line_and_goes_on( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def run(self: Any) -> int: + self.mode = "unknown" + raise KeyboardInterrupt + + _stub_attach(monkeypatch, run) + with pytest.raises(KeyboardInterrupt): + await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) + assert "the mode commit's outcome is unknown" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_stray_read_names_a_refused_table_and_reads_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.cli import cell_ch, doctor_cell + from server.core import pg + + async def topology(_ch: Any) -> Any: + return cell_ch.Topology("", 0) + + async def no_tables(*_a: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + async def sizes(*_a: Any) -> dict[str, int]: + return {} + + pg_rows = [{"name": "loom.a_unindexed", "n": 1}, {"name": "loom.b_data", "n": 2}] + conn = FakeConn([("FROM loom.tenancy", []), ("pg_class c", pg_rows)]) + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def pg_orgs(table: str, _budget: Any) -> list[str]: + if table == "loom.a_unindexed": + raise Refused([Refusal(f"postgres {table}", "has no plain btree index led by org")]) + return ["stray-org"] + + monkeypatch.setattr(cell_ch, "read_topology", topology) + monkeypatch.setattr(cell_scan, "list_clickhouse", no_tables) + monkeypatch.setattr(doctor_cell, "_table_sizes", sizes) + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(None, 60)} + assert lines["cell_stray_orgs"] == ("WARN", "postgres loom.b_data: stray-org") + assert lines["cell_stray_orgs_unread"] == ( + "WARN", + "not read: postgres loom.a_unindexed: has no plain btree index led by org", + ) + assert "cell_stray_orgs_unchecked" not in lines From a64cbb28b5ef130e72f67ccad16657b8edeca930 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 009/181] The attach docs say which role attach exempts, and the trigger gate's figures say what they establish cells.md and the runbook state that attach exempts the role it runs as and doctor does not. results.md gives the marked batch's per-round changes and their mean, says the delivery-pass gate is met on the mean but not established by three rounds alone, and that the memory.set p99 gate is unresolved locally. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +++++---- docs/benchmarks/results.md | 19 +++++++++++-------- docs/operations/cell-attach.md | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 3ea7ddab80..246da73b77 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1007,10 +1007,11 @@ It refuses, before any write, naming each: without one, or a role that may not write the durable read's logical message; a role of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with `max_prepared_transactions` above 0; -- **the tenancy-privilege check**: a login role, other than attach's own (superusers, - and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and - those named with `--pg-allow-role`, that holds or reaches through any membership a right - to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or +- **the tenancy-privilege check**: a login role, other than attach's own (the role attach + runs as, which the operator runs it with; superusers; and a role that itself owns + `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and those named with + `--pg-allow-role`, that holds or reaches through any membership a right to make an org + held or lower the mode: `INSERT` on any column of `loom.tenancy` or `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system one (a function there can shadow one the servers call unqualified on their marked diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 81e099d3c8..4814bd7f5f 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1241,14 +1241,17 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch, which is what a delivery pass of a cell's own processes writes, shows no -cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. -The `memory.set` figures move by more between rounds of one configuration than any trigger -cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms -here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no -consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the -host's noise, and the statement-level figures above bound the marked cost at under a -microsecond a row. An unmarked session pays about 11 µs a row, as above. +The marked batch is what a delivery pass of a cell's own processes writes. On minus off +within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are +9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is +met on the mean, but one round of three was 13% slower and three rounds give no paired bound +(methodology's paired bootstrap is for per-item labels), so this run does not establish the +gate on its own; the statement-level figures above do, at under a microsecond a row against +about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one +configuration than any trigger cost: on this loaded host the p99 differences within a round +(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same +measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is +unresolved locally. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index c51541d06d..481b9a95ef 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -135,10 +135,11 @@ procedure as an over-long org's. `loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an -org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or -a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a -writer added after `serving` passes attach's checks, which ran once; doctor is how it is -found. +org held or lower the mode (`cell_tenancy_privilege`). Attach exempts the role it runs as, +the operator's; doctor does not exempt the role it runs as, so a CI or monitoring role +holding such a right is named. It warns too on a role or database setting or a subscription +that passes an unchecked writer (`cell_unchecked_writers`). A grant or a writer added after +`serving` passes attach's checks, which ran once; doctor is how it is found. ## Related From 84e64603095fe8c2bf47cab47998209ce1c08fa6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 010/181] Every attach try runs once whatever --wait has left, the mode reads unknown only while a try's UPDATE is out, and a busy migration lock is named Each trigger and mode-commit try gets a lock wait of at least TRY_FLOOR_MS before --wait is judged spent, so --wait 0 is one try of each wait with no retry, and an idle cell attaches with it. The run's mode turns unknown just before a try sends its UPDATE and back to recording when that try rolls back. A trigger try that finds the migration lock busy names that lock's holder, in the retry line and in the refusal. The step-4 watch task is awaited after its cancel. The ATTACH_LOCK_KEY comment says who holds the lock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 24 +++- server/cli/cell_attach_pg.py | 74 ++++++++--- server/core/cell.py | 6 +- tests/integration/test_cell_attach.py | 17 ++- tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_attach.py | 176 ++++++++++++++++++++++++-- 7 files changed, 260 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 23449ccff9..8ed70e2d37 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -30,6 +30,7 @@ import argparse import asyncio +import contextlib import json import sys import time @@ -453,16 +454,22 @@ async def commit_mode(self) -> None: from server.core import pg await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - # From the mode transaction on the mode may have risen: only the durable read can say. - self.mode = "unknown" + + def mark(mode: str) -> None: + # `unknown` from a try's UPDATE on: only the durable read can say whether it + # landed; `recording` again when that try rolled back. + self.mode = mode + try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark ) except Refused: raise - except Exception as e: # noqa: BLE001 — the durable read says whether it landed + except Exception as e: + if self.mode != "unknown": + raise # no try was in flight: nothing to decide self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -573,6 +580,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: raise Refused(writers) finally: task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp @@ -757,7 +766,12 @@ async def _run(ch: Any) -> int: def add_attach_parser(csub: Any) -> None: p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") - p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--wait", + type=int, + default=_WAIT_S, + help="Seconds of wall time the waits share; 0 runs one try of each, with no retry", + ) p.add_argument( "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 9c6e4708d6..0ca9fefcc1 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -24,6 +24,8 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval +# The least lock wait a try gets, however little of --wait is left. +TRY_FLOOR_MS = 100 # not a setting: one try's floor # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound @@ -67,8 +69,10 @@ def counting(self) -> Iterator[None]: self._since = None def bound_ms(self, most_ms: int) -> int: - """A lock timeout for one try: `most_ms`, or what is left when that is less.""" - return min(most_ms, int(self.left() * 1000)) + """A lock timeout for one try: `most_ms`, or what is left when that is less, and + never under `TRY_FLOOR_MS`: every try runs once, so `--wait 0` is one try of each + wait with no retry, and only after a try is the wait judged spent.""" + return max(TRY_FLOOR_MS, min(most_ms, int(self.left() * 1000))) async def pause(self, seconds: float = POLL_S) -> bool: """Wait `seconds` (at most what is left); False once none is left.""" @@ -89,6 +93,13 @@ def _split(key: int) -> str: ) +# Who holds an advisory lock (`where` from `_split`), never a query's text. +_ADVISORY_HOLDER = """ +SELECT l.pid, a.application_name, a.backend_start +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {where} AND l.granted +""" + _LOCK_HOLDER = f""" SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid @@ -467,6 +478,11 @@ async def _ensure_table( "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): + if outcome.holders.startswith(_MIGRATION_HELD): + return Refusal( + f"postgres loom.{table}", + f"its triggers were not created within --wait: {outcome.holders}", + ) return _busy_refusal(table, await cell_triggers.holders(conn, table)) # Said again only when the holders change: a retry a second is otherwise silent. if outcome.holders != said: @@ -474,29 +490,45 @@ async def _ensure_table( log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") +_MIGRATION_HELD = "the migration lock" + + +async def _migration_holder(conn: Any, key: int) -> str: + """`the migration lock (held by …)`, naming the session that holds it.""" + try: + rows = await conn.fetch(_ADVISORY_HOLDER.format(where=_split(key))) + except Exception as e: # noqa: BLE001 — naming the holder is best effort + return f"{_MIGRATION_HELD} (its holder not read: {type(e).__name__})" + held = "; ".join( + f"pid {r['pid']}, {r['application_name'] or 'no name'}, since {r['backend_start']}" + for r in rows + ) + return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" + + async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock - wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" + wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the + try floor. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) - if migration_ms <= 0: - return cell_triggers.Outcome("busy") - await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return cell_triggers.Outcome("busy") # a boot migrating: the next try + # A boot migrating: the next try. + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) raise finally: await conn.execute("RESET lock_timeout") try: - lock_ms = wait.bound_ms(most_ms) - if lock_ms <= 0: - return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, ) finally: await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) @@ -543,21 +575,29 @@ async def commit_enforced( ready: Callable[[], Any], wait: Wait, log: Callable[[str], None], + mark: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before each try (False skips the try while a process boots). After a failed try the in-flight - mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; - whether the commit landed is the durable read's to say.""" + mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try + sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` + runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - lock_ms = wait.bound_ms(_MODE_LOCK_MS) - if lock_ms > 0 and await ready(): - if await _mode_try(conn, lock_ms): + if await ready(): + mark("unknown") + try: + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + except Refused: + mark("recording") # the UPDATE's transaction rolled back + raise + if done: return + mark("recording") log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..37e20f99cb 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -62,9 +62,9 @@ # Postgres's btree entry limit (about 2.7 KB), so every org a cell records has a # placement the directory can index. ORG_MAX_BYTES = 2048 # not a setting: cells and the router judge one limit -# The advisory lock on a cell's Postgres that attach will hold, and prepare and update once -# they take it, and that the troubleshooting page's full-state cleanup takes before the -# migration lock. Positive: the page's holder query splits it into two `oid` halves. +# The advisory lock on a cell's Postgres that attach and prepare hold for their run, and that +# the troubleshooting page's full-state cleanup takes before the migration lock. Positive: +# the page's holder query splits it into two `oid` halves. ATTACH_LOCK_KEY = 0x6C6F6F6D_61746368 # "loomatch" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index d77592d1dc..65064ab365 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -482,7 +482,9 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await cell_attach_pg.commit_enforced( + conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + ) await task assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] @@ -745,3 +747,16 @@ async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_store await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_wait_0_attaches_an_idle_cell(cell_stores, capsys) -> None: + """--wait 0 is one try of each wait, no retry: an idle cell still reaches serving.""" + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "w0" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + capsys.readouterr() + assert await _attach(stores, network, wait=0) == 0, capsys.readouterr().err + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert org in await _placed() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 3e88dbafca..4e45db4285 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -461,7 +461,7 @@ ], "loom cell attach": [ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 3d597c1e03..a31a862106 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13032,7 +13032,7 @@ "nargs": null, "type": "builtins:int", "required": false, - "help": "Seconds the waits share", + "help": "Seconds of wall time the waits share; 0 runs one try of each, with no retry", "metavar": null }, { diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b3db51ee02..429c2fcfb7 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -250,7 +250,7 @@ async def ready() -> bool: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -290,7 +290,9 @@ async def busy(*_a: Any, **_kw: Any) -> Any: wait = apg.Wait(10, clock) logged: list[str] = [] (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) - assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # The last try ran with the floor's lock wait, after --wait was all but spent. + assert "not taken within --wait" in refusal.reason + assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 # Said once per change of holders, not once a try. assert logged == [ "loom.scope_generations: its lock is busy (pid 7); retrying", @@ -654,7 +656,9 @@ async def ready() -> bool: return True logged: list[str] = [] - await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + ) assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 @@ -692,7 +696,9 @@ async def booting() -> bool: return False with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + await apg.commit_enforced( + conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + ) assert not any("LOCK TABLE" in s for s in conn.executed) @@ -789,8 +795,8 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*_a: Any) -> None: - pass # the commit landed; its answer may not have + async def landed(*a: Any) -> None: + a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} @@ -839,22 +845,166 @@ async def unread(_roles: Any) -> Any: @pytest.mark.asyncio -async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: - """A migration lock that took the rest of --wait leaves no DDL try.""" +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + """A migration lock that took the rest of --wait leaves the DDL its floor: one try, never + the full 100 ms when less is left, never none.""" + from server.core import cell_triggers def migration_lock(*_a: Any) -> str: - clock.now += 2.0 # the rest of --wait + clock.now += 1.95 # all but 50 ms of --wait return "OK" + asked: list[str] = [] + + async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: + asked.append(lock_timeout) + return cell_triggers.Outcome("applied") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) - assert outcome.answer == "busy" - assert not any( - "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + assert outcome.answer == "applied" + assert asked == [f"{apg.TRY_FLOOR_MS}ms"] + + +@pytest.mark.asyncio +async def test_a_busy_migration_lock_is_named_as_such( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + class _Busy(Exception): + sqlstate = "55P03" + + holder = [{"pid": 42, "application_name": "loom", "backend_start": "t0"}] + conn = FakeConn( + [ + ("pg_advisory_lock", _Busy()), + ("l.locktype = 'advisory'", holder), + ("deadlock_timeout", 1000), + ] ) + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + named = "the migration lock (held by pid 42, loom, since t0)" + assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] + assert refusal.reason == f"its triggers were not created within --wait: {named}" + + +@pytest.mark.asyncio +async def test_wait_0_runs_one_try_of_each_wait( + clock: _Clock, monkeypatch: pytest.MonkeyPatch +) -> None: + """On an idle database --wait 0 commits the mode and creates the triggers in one try + each, and the heartbeats pass when nothing boots.""" + from server.core import cell_triggers + + conn = FakeConn( + [ + ("LOCK TABLE", "LOCK TABLE"), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + marks: list[str] = [] + + async def ok() -> bool: + return True + + async def nothing() -> None: + pass + + await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + assert marks == ["unknown"] + assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def applied(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("applied") + + async def no_defects(_c: Any) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + monkeypatch.setattr(cell_triggers, "ensure_table", applied) + monkeypatch.setattr(cell_triggers, "missing", no_defects) + ddl = FakeConn([("deadlock_timeout", 1000)]) + assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + + async def read() -> list[beats.Beat]: + return [_beat("a")] + + assert (await beats.settle(read, _judge, apg.Wait(0, clock))).live == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refused_before_any_try", [True, False]) +async def test_a_mode_commit_refused_without_a_landed_try_says_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + clock: _Clock, + refused_before_any_try: bool, +) -> None: + """The closing line says recording when no try sent its UPDATE (the heartbeats refused + before one) and when every try timed out and rolled back.""" + + def lock(*_a: Any) -> str: + raise _Timeout() + + conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def ready(_self: Any) -> bool: + if refused_before_any_try: + raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) + return True + + async def confirm(_self: Any) -> None: + pass + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.wait = apg.Wait(2, clock) + await self.commit_mode() + return 0 + + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) + monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) + monkeypatch.setattr(cell_attach.Run, "run", run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + @pytest.mark.asyncio async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: From 5615d41a038625da621b7d25431284b6e8acdb12 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 011/181] The attach docs say what --wait 0 does, and cells.md keeps each code span on one line inside its list items Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 185 +++++++++++++++++---------------- docs/operations/cell-attach.md | 27 ++--- 2 files changed, 108 insertions(+), 104 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 246da73b77..53495c7a1d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1099,12 +1099,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): `enforced`; then `attach/serving`. `--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try -included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each -full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints -what would be placed and skipped and the scan's time, and writes nothing; the router's -checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: -`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's -Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. +included and never longer than what is left, but for a floor of 100 ms: every try runs once +before the wait is judged spent, so `--wait 0` is one try of each wait with no retry; +`--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. +`--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the +scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a +`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode +refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, +and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1487,58 +1489,58 @@ An integration test with two cells and a router in CI's Postgres job: between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token - revocation) runs, also with Postgres down on a deployment never prepared, and a `loom -cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres - down; after `loom cell credential --cell --next` both credentials place and read - state, `--finish` refuses the old credential, for a cell finished before its pods hold the - new one the router refuses its calls with 401, its request answers 503 writing nothing, - and it recovers by recording the old hash again, `--revert` drops a pending `next`, which - is then refused, and the old credential keeps placing, and a second `--next` while one is - pending and a hash another cell records as current or next are refused; `loom cell limit` - and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is - unavailable; concurrent placements at one under `placements_max` across two router - replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set - refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a - later cell, and on a first cell in `recording` runs and records the org; attach with - `placements_max` below the orgs the first cell holds places them all, and a later cell's - attach past its quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step 2 - and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a - process whose cache still reads `recording` sends a new org's request to the placement - call, which a closed cell answers 503; a CLI command whose insert finds `enforced` - refuses, naming the org, and makes no placement call; a refusal after step 3's write - leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; - each ordering of attach's write of the mode with a recording insert holds as stated above - (running, starting during the write, a stream of overlapping inserts each shorter than a - try, with attach setting `enforced` within a try or two, held across several tries, held - past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut - off by its timeout, and a commit whose connection is lost while it waits for the standby, - which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran - before `enforced` committed, on attach's role or a second role, holds step 4 until it ends - and its write of an unheld org is then placed, one whose first statement ran after is - refused by the trigger, a transaction that starts after attach takes the lock but before - its commit is visible holds step 4, a transaction opened in `recording` that outlives one - run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom - table does not hold step 4 and a background worker with a transaction open since - `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` - raised during the wait) makes step 4 refuse at once, naming it, with the mode still - `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write - transaction open in a second database on the same server holds step 4 and is named, a long - read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on - does not hold it, nor does a base backup running across it, a write held open on a - `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a long - logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past - `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first - cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 - refuses before step 1 with the mode still `recording`, while a later cell's attach does - not require either; a missing or disabled org trigger is refused by attach and named by - `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted - is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset - is unready and runs no worker, and a CLI command declaring an org with it unset is - refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` - cell is refused, also under `session_replication_role = replica`, and a missing or - disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + revocation) runs, also with Postgres down on a deployment never prepared, and a + `loom cell` store command refuses; `loom cell limit --closed` succeeds with the cell's + Postgres down; after `loom cell credential --cell --next` both credentials place + and read state, `--finish` refuses the old credential, for a cell finished before its pods + hold the new one the router refuses its calls with 401, its request answers 503 writing + nothing, and it recovers by recording the old hash again, `--revert` drops a pending + `next`, which is then refused, and the old credential keeps placing, and a second `--next` + while one is pending and a hash another cell records as current or next are refused; + `loom cell limit` and `loom cell credential --next`, `--finish` and `--revert` refuse + while the directory is unavailable; concurrent placements at one under `placements_max` + across two router replicas write one; a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a first cell once it is `enforced` + (`attaching` included) and on a later cell, and on a first cell in `recording` runs and + records the org; attach with `placements_max` below the orgs the first cell holds places + them all, and a later cell's attach past its quota or closed refuses, naming the org; a + first cell with `--closed` and `placements_max` set before step 2 attaches, and an org a + request records between step 2 and `enforced` is placed by step 4 with no 503; after step + 3's write of the cell's mode, a process whose cache still reads `recording` sends a new + org's request to the placement call, which a closed cell answers 503; a CLI command whose + insert finds `enforced` refuses, naming the org, and makes no placement call; a refusal + after step 3's write leaves the cell `enforced` and `attaching`, where a new org is placed + through the gate; each ordering of attach's write of the mode with a recording insert + holds as stated above (running, starting during the write, a stream of overlapping inserts + each shorter than a try, with attach setting `enforced` within a try or two, held across + several tries, held past `--wait` (attach refuses, the cell still `recording`), a stalled + attach, a commit cut off by its timeout, and a commit whose connection is lost while it + waits for the standby, which the durable read decides); a `REPEATABLE READ` transaction + whose first statement ran before `enforced` committed, on attach's role or a second role, + holds step 4 until it ends and its write of an unheld org is then placed, one whose first + statement ran after is refused by the trigger, a transaction that starts after attach + takes the lock but before its commit is visible holds step 4, a transaction opened in + `recording` that outlives one run's `--wait` holds the rerun's step 4 after a refusal, an + autovacuum worker on a loom table does not hold step 4 and a background worker with a + transaction open since `recording` does, a prepared transaction (after a restart with + `max_prepared_transactions` raised during the wait) makes step 4 refuse at once, naming + it, with the mode still `enforced` and a rerun refused until it is resolved and the + setting is back to 0, a write transaction open in a second database on the same server + holds step 4 and is named, a long read on the synchronous standby, or on a second standby, + with `hot_standby_feedback` on does not hold it, nor does a base backup running across it, + a write held open on a `replication=database` connection holds step 4, and so does a + `REPEATABLE READ` transaction on one, opened in `recording` and writing nothing before + `enforced`, a long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's + database past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and + the first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` + above 0 refuses before step 1 with the mode still `recording`, while a later cell's attach + does not require either; a missing or disabled org trigger is refused by attach and named + by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was + deleted is refused and the state named by `loom doctor`; on that cell a pod with + `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an org with + it unset is refused, as a pod is; updating, deleting or truncating the `first` row on an + `enforced` cell is refused, also under `session_replication_role = replica`, and a missing + or disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer request and runs no worker pass, and serves again after a rerun of `loom cell prepare` writes the row back while it runs; a cell the directory registers with no state, tenancy @@ -1548,12 +1550,12 @@ cell` store command refuses; `loom cell limit --closed` succeeds with the cell's rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, has the function refuse an unheld org (a boot creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a - Postgres-only restore of an attached first cell to before its prepare is named by `loom -doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell -update` on an attached cell, with processes on both values and `loom.cell_self` lost, - writes both values back and keeps both sets of processes serving, and one before attach - refuses the two configurations; a prepare rerun on an attached cell whose `current` is - missing and whose pods' configuration differs from the directory row refuses, and one + Postgres-only restore of an attached first cell to before its prepare is named by + `loom doctor` and closed by `loom cell restore`; a prepare rerun during a pending + `loom cell update` on an attached cell, with processes on both values and `loom.cell_self` + lost, writes both values back and keeps both sets of processes serving, and one before + attach refuses the two configurations; a prepare rerun on an attached cell whose `current` + is missing and whose pods' configuration differs from the directory row refuses, and one before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a credential hash that is neither the row's `current` nor its `next` is refused; a prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a @@ -1561,33 +1563,34 @@ update` on an attached cell, with processes on both values and `loom.cell_self` creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the row back; a later cell whose mode row is set to `recording` records nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as - in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom -cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` - set refuses, naming it, and the router refuses its state read; a build N-1 process's - `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed - and write no tenancy row, and on the first cell in `recording` a boot creates no org - trigger, and one created by hand reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), an - `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one - that hits an existing row of it is not, a marked session's every write passes, and the - triggers on an `enforced` cell cover every table erasure classifies with an `org` column - but the three exclusions; and an org-keyed write while attach holds the mode table's lock - returns at once, with new-org requests answering within their deadline and mode-cache - reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` - with its directory row `enforced`, and the rerun runs step 2 again; a - `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom -doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with - the directory and the stores unreachable; prepare and `--next` refuse an argument that is - not 64 lowercase hex characters, the credential and its bare 128-character secret - included, storing nothing; orgs a first cell records in `recording` with no data are - listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at - each router replica, `loom doctor` names the cell as one with no quota, and a call refused - past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its - retry at that replica, with no other call of the cell admitted between, is admitted; a - cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or - CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the - authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep - working through the cutover, a logout through the shared host revokes its refresh token; + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, + `loom cell prepare` names the row, a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses its state read; a + build N-1 process's `memory.set` and delivery pass on a never-prepared deployment after + N's migration succeed and write no tenancy row, and on the first cell in `recording` a + boot creates no org trigger, and one created by hand reads the state and looks nothing up, + a write under `session_replication_role = replica` for an unheld org is skipped (origin + triggers), an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is + refused and one that hits an existing row of it is not, a marked session's every write + passes, and the triggers on an `enforced` cell cover every table erasure classifies with + an `org` column but the three exclusions; and an org-keyed write while attach holds the + mode table's lock returns at once, with new-org requests answering within their deadline + and mode-cache reads returning at once throughout; a refusal at step 3's lock leaves the + cell `recording` with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by + `loom doctor`; `loom cell credential new` prints a credential of that form and its + SHA-256, with the directory and the stores unreachable; prepare and `--next` refuse an + argument that is not 64 lowercase hex characters, the credential and its bare + 128-character secret included, storing nothing; orgs a first cell records in `recording` + with no data are listed by `loom doctor`; a placement flood with no quota set is held to + the rate limit at each router replica, `loom doctor` names the cell as one with no quota, + and a call refused past it at one replica answers 503 with a `Retry-After` of at most 60 + s, after which its retry at that replica, with no other call of the cell admitted between, + is admitted; a cell's first prepare, with no directory row yet, passes the deployment-id + check; a pod or CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is + refused by the authority check; the first cell's existing tokens, refresh tokens and OAuth + clients keep working through the cutover, a logout through the shared host revokes its + refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 481b9a95ef..b713aefe0d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -59,14 +59,14 @@ Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` o `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role -of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the -writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, -`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT -QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings -profile or quota capping the rows or bytes it reads, and the insert into -`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the -bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of +`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, +`SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and +`READ ON REMOTE`), no settings profile or quota capping the rows or bytes it reads, and the +insert into `loom._migration_lock` that writes the probe nonce. The CLI needs write access +to the bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. ## Run it @@ -80,13 +80,14 @@ loom cell attach --cell \ `--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs per source table, the ones it would skip, and whether the scan fit in half of `--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, -the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object -and replica that may write the configured databases with `--ch-allow-user`, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object and +replica that may write the configured databases with `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each -list is audited by the router. `--wait` (default 600 s) bounds the waits together; -`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of -it, since step 4 reads the same tables again. +list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; +every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs +one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, +and step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it From b0f694b7e52bb17475cedb5f005de18d353dc435 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:05:39 +0000 Subject: [PATCH 012/181] Bound cell attach's mode commit and name booting processes Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +- docs/operations/cell-attach.md | 6 +- server/cli/cell_attach.py | 4 +- server/cli/cell_attach_pg.py | 34 +++++--- tests/integration/test_cell_attach.py | 4 +- tests/unit/test_cell_attach.py | 111 +++++++++++++++++++++++--- 6 files changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 53495c7a1d..f32815b635 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1065,8 +1065,10 @@ Then, in order (the step numbers are the ones the rest of this page uses): backed out. - **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock (only + cheap heartbeat refusals run before each try (a booting process skips the try). Each try, + including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and + a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell `enforced` and `attaching`, and a rerun resumes from the durable read. diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index b713aefe0d..ec6b6354ec 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -86,8 +86,10 @@ replica that may write the configured databases with `--ch-allow-user`, Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs -one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, -and step 2 must take at most half of it, since step 4 reads the same tables again. +one try of each wait with no retry. The final heartbeat checks get at least two heartbeats +plus the router timeout, with one extension for a process first seen booting; this floor +is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and +step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8ed70e2d37..6ef819796a 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -442,13 +442,13 @@ async def triggers(self) -> None: # -- step 3 ------------------------------------------------------------------------ - async def ready_for_commit(self) -> bool: + async def ready_for_commit(self) -> list[str]: """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, a booting process skips the try.""" verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) if verdict.refusals: raise Refused(verdict.refusals) - return not verdict.booting + return verdict.booting async def commit_mode(self) -> None: from server.core import pg diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 0ca9fefcc1..60196363af 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -29,9 +29,13 @@ # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# A trigger try yields quickly to active writers. +_TRIGGER_LOCK_MS = 100 # not a setting: one try's bound # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait +# COMMIT's standby wait is outside Postgres's statement timeout. +_MODE_STANDBY_S = 5.0 # not a setting: one try's standby allowance # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} @@ -465,7 +469,7 @@ async def _ensure_table( yielded = False said = "" while True: - most_ms = 100 + most_ms = _TRIGGER_LOCK_MS holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True @@ -572,7 +576,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( conn: Any, confirm_lock: Callable[[], Any], - ready: Callable[[], Any], + ready: Callable[[], Awaitable[list[str]]], wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], @@ -581,17 +585,17 @@ async def commit_enforced( session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before - each try (False skips the try while a process boots). After a failed try the in-flight + each try (the booting process ids skip the try). After a failed try the in-flight mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - if await ready(): - mark("unknown") + booting = await ready() + if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -601,6 +605,13 @@ async def commit_enforced( log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): + if booting: + raise Refused( + [ + Refusal(f"process {p}", "still booting at the end of --wait") + for p in booting + ] + ) raise Refused( [ Refusal( @@ -612,15 +623,18 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int) -> bool: - """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" +async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + """One try including COMMIT, bounded by its lock and statement waits plus the standby + allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with conn.transaction(): + async with asyncio.timeout(bound_s), conn.transaction(): await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 65064ab365..f6535cdb40 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -469,8 +469,8 @@ async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None await _prepare(cell_stores, _Network()) logged: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async with pg.dedicated() as holder, pg.dedicated() as conn: tx = holder.transaction() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 429c2fcfb7..95fc006333 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json from collections.abc import Callable from dataclasses import dataclass, field @@ -244,8 +245,8 @@ def lock(sql: str, *_a: Any) -> str: async def nothing() -> None: pass - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait @@ -652,8 +653,8 @@ def lock(*_a: Any) -> str: async def confirm() -> None: confirmed.append(True) - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] logged: list[str] = [] await apg.commit_enforced( @@ -684,24 +685,108 @@ async def confirm() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( clock: _Clock, + monkeypatch: pytest.MonkeyPatch, + wait_s: int, ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.own_fp = (CURRENT, NEXT) + + async def read(_ch: Any) -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name=None)] + + monkeypatch.setattr(beats, "read_beats", read) async def confirm() -> None: pass - async def booting() -> bool: - return False - - with pytest.raises(Refused, match="not taken within --wait"): + with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + conn, + confirm, + run.ready_for_commit, + apg.Wait(wait_s, clock), + lambda _t: None, + lambda _m: None, ) + assert [str(r) for r in error.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: still booting at the end of --wait", + ] assert not any("LOCK TABLE" in s for s in conn.executed) +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s,bound_s", [(0, 8.1), (10, 10.0)]) +async def test_the_mode_try_bounds_a_commit_that_never_answers( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float +) -> None: + class _StalledCommit(FakeConn): + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + assert not exc[0] + conn.executed.append("COMMIT") + await asyncio.Future() + + return _Tx() + + conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) + bounds: list[float] = [] + timeout = asyncio.timeout + + def bounded(seconds: float) -> Any: + bounds.append(seconds) + return timeout(0) + + monkeypatch.setattr(apg.asyncio, "timeout", bounded) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + apg.commit_enforced( + conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + ), + timeout=0.2, + ) + assert conn.executed[-1] == "COMMIT" + assert bounds == [bound_s] + assert marks == ["unknown"] + + +@pytest.mark.asyncio +async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: + conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(KeyboardInterrupt): + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + ) + assert conn.executed[-1] == "ROLLBACK" + assert marks == [] + + @pytest.mark.asyncio async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( monkeypatch: pytest.MonkeyPatch, @@ -921,8 +1006,8 @@ async def test_wait_0_runs_one_try_of_each_wait( ) marks: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async def nothing() -> None: pass @@ -975,10 +1060,10 @@ def lock(*_a: Any) -> str: async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} - async def ready(_self: Any) -> bool: + async def ready(_self: Any) -> list[str]: if refused_before_any_try: raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) - return True + return [] async def confirm(_self: Any) -> None: pass From fefe1e04f092064190d35752f78adc42a42a0743 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:18:57 +0000 Subject: [PATCH 013/181] Require every cell probe grant before attach Check each probe privilege separately because a comma-separated privilege list accepts any one of them. Run the attach integration tests in the Postgres matrix and limit their heartbeat reads to the test's own process IDs while retaining the real queries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/cli/cell_attach_pg.py | 5 ++- tests/integration/test_cell_attach.py | 50 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 564393a0f7..22342c2cab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -898,6 +898,7 @@ jobs: tests/integration/test_router_directory_pg.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py + tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 60196363af..583d99775d 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -397,7 +397,10 @@ async def session_passes() -> bool: ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', - has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + has_table_privilege('loom.cell_probe', 'SELECT') + AND has_table_privilege('loom.cell_probe', 'INSERT') + AND has_table_privilege('loom.cell_probe', 'UPDATE') + AND has_table_privilege('loom.cell_probe', 'DELETE')), ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) ) v(need, held) WHERE NOT held diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index f6535cdb40..aaa48b8f0e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -102,6 +102,21 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str ch = get_client() await ch.connect() ids: list[str] = [] + real_live, real_recent = instances.read_live, instances.read_recent + + async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, Any]]: + rows = await real_live(ch, all_deployments=all_deployments) + return [r for r in rows if r["instance_id"] in ids] + + async def read_recent( + ch: Any, *, window_s: float, since: str | None = None + ) -> list[dict[str, Any]]: + rows = await real_recent(ch, window_s=window_s, since=since) + return [r for r in rows if r["instance_id"] in ids] + + # The integration server shares ClickHouse and reports its own process heartbeats. + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: await pg_migrate.migrate() await pg.execute( @@ -669,6 +684,41 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_attach_requires_every_probe_privilege(cell_stores) -> None: + role = f"it_probe_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + privileges = ("SELECT", "INSERT", "UPDATE", "DELETE") + grants = [ + ("SELECT",), + *(tuple(p for p in privileges if p != missing) for missing in privileges), + privileges, + ] + observed: dict[tuple[str, ...], list[str]] = {} + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + for held in grants: + await conn.execute(f"REVOKE ALL ON loom.cell_probe FROM {role}") + await conn.execute(f"GRANT {', '.join(held)} ON loom.cell_probe TO {role}") + await conn.execute(f"SET ROLE {role}") + refusals = await cell_attach_pg.own_role_refusals(conn, []) + await conn.execute("RESET ROLE") + observed[held] = [r.reason for r in refusals if "loom.cell_probe" in r.reason] + assert observed == { + held: ( + [] + if held == privileges + else ["attach's role lacks SELECT, INSERT, UPDATE and DELETE on loom.cell_probe"] + ) + for held in grants + } + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( cell_stores, From bd4cda33fd5d2bbbbc26812b1be8b3bec9c0212a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:29:02 +0000 Subject: [PATCH 014/181] Keep cell attach deadlines from waiting on rollback Manage mode and direct-session transactions explicitly so a canceled statement cannot start an unbounded rollback. Roll back ordinary mode statement errors within the try's deadline; terminate synchronously when a try ends with an error or cancellation. Signed-off-by: zlareb1 --- server/cli/cell_attach_pg.py | 64 +++++++++++------- tests/unit/test_cell_attach.py | 116 ++++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 583d99775d..d58b553d7a 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -122,10 +122,18 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: startup parameters drops it). Behind a transaction pooler a session-level lock would outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" - async with conn.transaction(): - first = await conn.fetchrow(_IDENTITY) - async with conn.transaction(): - second = await conn.fetchrow(_IDENTITY) + identities = [] + try: + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() + except BaseException: + # A cancelled read must not wait for a transaction context's rollback. + conn.terminate() + raise + first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): out.append( @@ -627,31 +635,39 @@ async def commit_enforced( async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: - """One try including COMMIT, bounded by its lock and statement waits plus the standby - allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + """One try including rollback or COMMIT, bounded by its lock and statement waits plus + the standby allowance; False after a statement timeout rolls back, TimeoutError at + the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with asyncio.timeout(bound_s), conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + async with asyncio.timeout(bound_s): + tx = conn.transaction() + await tx.start() + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False + raise + await tx.commit() return True - except Refused: + except BaseException: + # Termination is synchronous: cancellation never starts an unbounded rollback. + conn.terminate() raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise - return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 95fc006333..065f3a455d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -473,14 +473,29 @@ def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await conn.execute("BEGIN") + + async def commit(self) -> None: + await conn.execute("COMMIT") + + async def rollback(self) -> None: + await conn.execute("ROLLBACK") + async def __aenter__(self) -> None: - conn.executed.append("BEGIN") + await self.start() async def __aexit__(self, *exc: Any) -> None: - conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + if exc[0]: + await self.rollback() + else: + await self.commit() return _Tx() + def terminate(self) -> None: + self.executed.append("TERMINATE") + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -663,6 +678,8 @@ async def ready() -> list[str]: assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 + assert conn.executed.count("ROLLBACK") == 1 + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio @@ -725,19 +742,11 @@ async def test_the_mode_try_bounds_a_commit_that_never_answers( monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float ) -> None: class _StalledCommit(FakeConn): - def transaction(self) -> Any: - conn = self - - class _Tx: - async def __aenter__(self) -> None: - conn.executed.append("BEGIN") - - async def __aexit__(self, *exc: Any) -> None: - assert not exc[0] - conn.executed.append("COMMIT") - await asyncio.Future() - - return _Tx() + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "COMMIT": + await asyncio.Future() + return result conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] @@ -763,11 +772,84 @@ async def ready() -> list[str]: ), timeout=0.2, ) - assert conn.executed[-1] == "COMMIT" + assert "COMMIT" in conn.executed + assert conn.executed[-1] == "TERMINATE" assert bounds == [bound_s] assert marks == ["unknown"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["LOCK TABLE", "UPDATE", "ROLLBACK"]) +async def test_a_mode_deadline_never_waits_for_rollback( + monkeypatch: pytest.MonkeyPatch, stall: str +) -> None: + class _StalledRollback(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "ROLLBACK" and sql.startswith("LOCK TABLE"): + raise _Timeout() + if sql.startswith(stall) or sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + marks: list[str] = [] + task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "mode deadline is stuck awaiting rollback" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + assert marks == (["unknown"] if stall == "UPDATE" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_read", [1, 2]) +@pytest.mark.parametrize("failure", ["cancel", "error"]) +async def test_a_failed_direct_session_read_never_waits_for_rollback( + failed_read: int, failure: str +) -> None: + class _StalledRollback(FakeConn): + reads = 0 + + async def fetchrow(self, sql: str, *args: Any) -> Any: + self.reads += 1 + if self.reads == failed_read: + if failure == "error": + raise ValueError("identity read failed") + await asyncio.Future() + return (1, "t0") + + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + task = asyncio.create_task(apg.direct_refusals(conn, "attach")) + await asyncio.sleep(0) + if failure == "cancel": + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "direct-session failure is stuck awaiting rollback" + if failure == "cancel": + assert task.cancelled() + else: + assert isinstance(task.exception(), ValueError) + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + @pytest.mark.asyncio async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) @@ -783,7 +865,7 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "ROLLBACK" + assert conn.executed[-1] == "TERMINATE" assert marks == [] From 3ba813ccd9b7c40f355e6bba537e5d7f1b6231c6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:44:09 +0000 Subject: [PATCH 015/181] Bound attach trigger attempts and scope gate cleanup to its run Give each trigger attempt a client deadline covering migration lock acquisition, DDL, commit, rollback and unlock. Use explicit transactions so cancellation terminates the session without waiting for rollback or masking the original error with lock cleanup. Give gate batch scopes a unique prefix per run so cleanup preserves existing rows. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- server/cli/cell_attach_pg.py | 47 ++++++++------ server/core/cell_triggers.py | 21 +++++- tests/unit/test_bench_cell_trigger_gate.py | 53 +++++++++++++++ tests/unit/test_cell_attach.py | 75 ++++++++++++++++++++++ tests/unit/test_cell_org_triggers.py | 14 ++-- 6 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_bench_cell_trigger_gate.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 4363a810d6..8797b4b8e7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -80,8 +80,9 @@ async def _batch(args: argparse.Namespace, marked: bool) -> float: t = time.perf_counter() await conn.execute( "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", args.org, + args.scope_prefix, uuid.uuid4().hex[:8], ) times.append((time.perf_counter() - t) * 1000) @@ -98,6 +99,7 @@ async def main() -> None: p.add_argument("--org", required=True) p.add_argument("--namespace", required=True) args = p.parse_args() + args.scope_prefix = f"gate-{uuid.uuid4().hex}-" try: for _ in range(ROUNDS): for on in (True, False): @@ -120,7 +122,9 @@ async def _clean(args: argparse.Namespace) -> None: conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) try: await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", + args.org, + args.scope_prefix, ) finally: await conn.close() diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d58b553d7a..2af085f86f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -31,6 +31,7 @@ _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound # A trigger try yields quickly to active writers. _TRIGGER_LOCK_MS = 100 # not a setting: one try's bound +_TRIGGER_TRY_MARGIN_S = 8.0 # not a setting: catalog, DDL and standby allowance # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait @@ -524,29 +525,37 @@ async def _migration_holder(conn: Any, key: int) -> str: async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the - try floor. A busy migration lock answers `busy` naming that lock's holder.""" + try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or + rollback. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + bound_s = ( + wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) + ) / 1000 + _TRIGGER_TRY_MARGIN_S try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) == "55P03": - # A boot migrating: the next try. - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + async with asyncio.timeout(bound_s): + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + try: + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) + return outcome + except BaseException: + # Closing the session releases its locks without asynchronous error cleanup. + conn.terminate() raise - finally: - await conn.execute("RESET lock_timeout") - try: - return await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - finally: - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 7a1ee5e5b7..1899a46bab 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -328,18 +328,32 @@ async def _run_ddl( import asyncpg for attempt in range(attempts): + rolled_back = False try: - async with conn.transaction(): + tx = conn.transaction() + await tx.start() + try: await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) + except Exception: + await tx.rollback() + rolled_back = True + raise + await tx.commit() return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: + if not rolled_back: + conn.terminate() + raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: + if not rolled_back: + conn.terminate() + raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) @@ -354,6 +368,11 @@ async def _run_ddl( held, ) return Outcome("busy", held) + except BaseException: + if not rolled_back: + # Cancellation must not wait for another statement on this session. + conn.terminate() + raise return Outcome("busy") diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py new file mode 100644 index 0000000000..89f32cddde --- /dev/null +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -0,0 +1,53 @@ +"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" + +from __future__ import annotations + +import argparse +import sqlite3 +from typing import Any + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.asyncio +async def test_batch_cleanup_preserves_preexisting_gate_scopes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = sqlite3.connect(":memory:") + db.execute("ATTACH DATABASE ':memory:' AS loom") + db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") + db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) + prefix = "gate-own-run-" + sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] + db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) + + class _Conn: + async def execute(self, sql: str, *args: Any) -> None: + sql = sql.replace( + "FROM generate_series(1, 1000) g", + "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " + "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", + ) + db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) + + async def close(self) -> None: + pass + + async def connect(*_a: Any, **_kw: Any) -> Any: + return _Conn() + + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate, "INSERTS", 1) + args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) + try: + await gate._batch(args, True) + assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 + await gate._clean(args) + assert ( + db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() + == sentinel + ) + finally: + db.close() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 065f3a455d..6d57cbb740 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -1038,6 +1038,81 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) +async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str +) -> None: + import asyncpg + + from server.core import cell_triggers + + class _Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "rollback" and sql == "CREATE TRIGGER stub": + raise asyncpg.LockNotAvailableError("busy") + needles = { + "migration": "SELECT pg_advisory_lock", + "reset": "RESET lock_timeout", + "ddl": "CREATE TRIGGER stub", + "commit": "COMMIT", + "rollback": "ROLLBACK", + "unlock": "SELECT pg_advisory_unlock", + } + if sql.startswith(needles[stall]) or (stall == "ddl" and sql == "ROLLBACK"): + await asyncio.Future() + return result + + async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: + return await cell_triggers._run_ddl( + conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False + ) + + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = _Stalled() + task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + done, _ = await asyncio.wait([task], timeout=0.2) + try: + assert task in done, f"trigger deadline is stuck at {stall}" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + if stall in {"migration", "ddl", "commit", "rollback"}: + assert apg._MIGRATION_UNLOCK not in conn.executed + finally: + for _ in range(3): + if task.done(): + break + task.cancel() + await asyncio.wait([task], timeout=0.01) + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["migration", "ddl"]) +async def test_a_failed_trigger_try_does_not_mask_its_error_with_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, failed: str +) -> None: + from server.core import cell_triggers + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + raise ValueError("DDL failed") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn( + [ + ("pg_advisory_lock", ValueError("migration failed") if failed == "migration" else "OK"), + ("RESET", RuntimeError("reset failed") if failed == "migration" else "OK"), + ("pg_advisory_unlock", RuntimeError("unlock failed")), + ] + ) + with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): + await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + assert conn.executed[-1] == "TERMINATE" + + @pytest.mark.asyncio async def test_a_busy_migration_lock_is_named_as_such( monkeypatch: pytest.MonkeyPatch, clock: _Clock diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index decb365521..769cfe1bb3 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -617,11 +617,17 @@ def __init__(self, error: Exception) -> None: self.error = error def transaction(self) -> Any: - @contextlib.asynccontextmanager - async def _tx() -> AsyncIterator[None]: - yield + class _Tx: + async def start(self) -> None: + pass - return _tx() + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass + + return _Tx() async def fetchrow(self, _sql: str, *_a: Any) -> dict[str, Any]: return { From 98fc3331237b6c9f5828e899b2abe1b7d6c2fdbb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:30:52 +0000 Subject: [PATCH 016/181] Operator placement durably records its directory winner and local tenancy Signed-off-by: zlareb1 --- docs/architecture/cells.md | 23 +- docs/reference/cli.md | 1 + server/cli/cell.py | 2 + server/cli/cell_place.py | 166 +++++++ server/core/cell.py | 7 +- server/router/app.py | 46 +- server/router/directory.py | 60 ++- tests/integration/test_cell_place.py | 238 +++++++++ tests/integration/test_router_directory_pg.py | 28 +- tests/unit/snapshots/cli_help.json | 8 +- tests/unit/snapshots/cli_tree.json | 99 ++++ tests/unit/test_cell_place.py | 466 ++++++++++++++++++ tests/unit/test_router.py | 4 +- tests/unit/test_router_attach.py | 15 +- 14 files changed, 1108 insertions(+), 55 deletions(-) create mode 100644 server/cli/cell_place.py create mode 100644 tests/integration/test_cell_place.py create mode 100644 tests/unit/test_cell_place.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index f32815b635..4c5fec9a85 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -754,13 +754,21 @@ read-only scope, lists any tenancy row the directory does not place. it, unless the cell is the first and its mode reads `recording`, and an org-free command never uses it; a leaked one is stopped by `loom cell limit --closed`, which needs none of the cell's stores, and replaced by `loom cell credential` (below). - `loom cell place --org ` (declared org-free: it writes the org's placement and - tenancy row, which is how an org comes to be held), with the router's operator credential, + `loom cell place --cell --org ` (declared org-free: it writes the org's + placement and tenancy row, which is how an org comes to be held), with the router's operator credential, places an org for a command that writes a new one (a bot, an import), through the router's placement API and in its order, as attach's placement of configured orgs does: it inserts the placement and reads back the winner, writes the cell's tenancy row only if the winner is this cell (otherwise it refuses and writes nothing), and reports success only once the - row is written, so a rerun is idempotent. For an org the directory does not place yet it + row commits synchronously. The CLI holds the attach lock on a direct marked Postgres + session, requires a full enforced local cell state and the full accepted configuration + fingerprint, and checks both again before inserting. A transactional durability message + makes even a no-op insert flush an earlier primary-only tenancy row to the standby. + Opening the session, local statements, COMMIT and cleanup have client deadlines derived + from `LOOM_PG_STATEMENT_TIMEOUT_MS`. An unknown outcome names rerunning the same command; + the permanent directory winner and idempotent local insert let that rerun finish. The + route checks directory state; a direct operator API caller is trusted to check local + Postgres mode as the CLI does. For an org the directory does not place yet it passes the placement API's gate (below): refused while the directory is unavailable, for a cell the gate does not accept, closed, or when the fingerprint of the CLI's own configuration of the cell (with the deployment id read from the stores it connects to) @@ -859,9 +867,12 @@ read-only scope, lists any tenancy row the directory does not place. safe, since the read-back makes it idempotent. For an org the directory already places, a cell's call is checked only for its credential (against the cells' hashes the router caches for `LOOM_ROUTER_DIRECTORY_CACHE_S`, default 5 s) and the cell's - rate, and a `loom cell place` only for the operator credential; both answer the winner - whatever the cell's state, quota or fingerprint, adding no placement (the winner is - public through the lookup anyway); a request refused past its cell's rate answers 503 + rate. An operator's `loom cell place` checks its deployment id and fingerprint against + the requested cell's locked row even for an org already placed there; an org placed + elsewhere answers that winner and the CLI refuses without inserting local tenancy. + Existing placements bypass closed and quota checks. Every successful call writes an + audit row before its synchronous commit, including a retry of an existing winner, so + the returned winner survives failover. A request refused past its cell's rate answers 503 with `Retry-After` (at most a minute), and its retry is admitted by the replica that answers it once the cell's calls there are under the rate. An org the cell already holds needs no call, so serving it never depends on the router. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 12bac9a65b..dd75540091 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -329,6 +329,7 @@ python scripts/codegen/docs_reference.py | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | +| `loom cell place` | Place an org in this cell and record its tenancy. | `--cell`, `--org`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | | `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | diff --git a/server/cli/cell.py b/server/cli/cell.py index 65f18990d8..c14927994c 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -460,7 +460,9 @@ def _add_cell_parser(sub: Any) -> None: remove.set_defaults(func=_cmd_remove) from server.cli.cell_attach import add_attach_parser + from server.cli.cell_place import add_place_parser from server.cli.cell_prepare import add_prepare_parser + add_place_parser(csub) add_prepare_parser(csub) add_attach_parser(csub) diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py new file mode 100644 index 0000000000..3820ecf2cc --- /dev/null +++ b/server/cli/cell_place.py @@ -0,0 +1,166 @@ +"""Directory-first operator placement, followed by a durable local tenancy row.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_pg as apg +from server.cli._ch import run_connected, writes +from server.cli.cell_ch import Refused +from server.core import cell, pg +from server.router.directory import PLACE_MAX_S +from server.settings import settings + + +class PlacementRefused(Exception): + """A placement cannot complete with this configuration or directory winner.""" + + +def _bound_s() -> float: + return settings.pg_statement_timeout_ms / 1000 + 1 + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if args.cell != settings.cell: + return f"place runs with the cell's own configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "place needs LOOM_STORAGE_AUTHORITY=postgres and the cell's LOOM_PG_DSN" + if cell.named_org(args.org) is None: + return "--org names a nonempty Postgres org" + return cell.org_name_refusal(args.org) + + +async def _identity(conn: Any, name: str) -> tuple[str, dict[str, Any]]: + """Read the full local cell state and this CLI's accepted configuration.""" + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + state, first = await cell.read_state_if_present(conn) + if state != "enforced" or first != name: + raise PlacementRefused( + f"local cell state is {state!r}, naming {first!r}; place needs enforced {name!r}" + ) + deployment = await conn.fetchval("SELECT deployment_id FROM loom.authority") + if not isinstance(deployment, str) or not deployment: + raise PlacementRefused("the local Postgres records no deployment id") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + mine = cell.fingerprint(deployment) + if own is None or not any( + cell.matches(mine, cell.json_object(own[k])) for k in ("current_fp", "next_fp") + ): + raise PlacementRefused("the CLI's configuration matches no local cell fingerprint") + return deployment, cell.as_value(mine) + + +async def _directory( + router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] +) -> None: + url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") + async with httpx.AsyncClient(trust_env=router.scheme == "https") as client: + answer = await client.post( + url, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, + timeout=httpx.Timeout(PLACE_MAX_S + 10.0, connect=5.0), + ) + posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) + if posted is None: + raise PlacementRefused(cell_cli.not_the_router(answer)) + if answer.status_code != 200: + raise PlacementRefused(f"the router refused ({answer.status_code}): {posted['error']}") + if posted.get("cell") != name: + raise PlacementRefused( + f"org {org!r} belongs to cell {posted.get('cell')!r}; no local tenancy row written" + ) + if posted.get("deployment_id") != deployment: + raise PlacementRefused("the router answered another deployment id") + + +async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: + """Hold the operator lock through directory placement and an acknowledged local commit. + Every local phase, including opening and closing the direct session, has a deadline.""" + cm = pg.dedicated(statement_timeout_ms=settings.pg_statement_timeout_ms) + conn = None + try: + async with asyncio.timeout(_bound_s()): + conn = await cm.__aenter__() + async with asyncio.timeout(_bound_s()): + if refusals := await apg.direct_refusals(conn, "place"): + raise Refused(refusals) + await apg.take_lock(conn, "place") + deployment, fingerprint = await _identity(conn, args.cell) + await _directory(router, args.cell, args.org, deployment, fingerprint) + async with asyncio.timeout(_bound_s()): + tx = conn.transaction() + await tx.start() + if not await apg.lock_held(conn): + raise PlacementRefused("the operator lock was lost") + if await _identity(conn, args.cell) != (deployment, fingerprint): + raise PlacementRefused( + "the local identity or configuration changed during placement" + ) + await conn.execute( + "INSERT INTO loom.tenancy (org) VALUES ($1) ON CONFLICT DO NOTHING", args.org + ) + # A no-op insert must also flush a primary-only predecessor to the standby. + await conn.execute(pg._DURABLE_MESSAGE) + await tx.commit() + finally: + if conn is not None: + try: + async with asyncio.timeout(_bound_s()): + await cm.__aexit__(None, None, None) + finally: + conn.terminate() + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + +def cmd_place(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell place: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell place: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + try: + result = await place(args, router) + except asyncio.CancelledError: + print( + "loom cell place: stopped (CancelledError); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + raise + except Exception as e: # noqa: BLE001 — an unknown outcome must name its idempotent retry + print( + f"loom cell place: stopped ({type(e).__name__}: {e}); " + "placement may have committed; rerun the same command", + file=sys.stderr, + ) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + return run_connected( + run, org_free=writes("places an org in the directory and records its local tenancy") + ) + + +def add_place_parser(csub: Any) -> None: + p = csub.add_parser("place", help="Place an org in this cell and record its tenancy.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--org", required=True, help="The org to place") + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_place) diff --git a/server/core/cell.py b/server/core/cell.py index 37e20f99cb..fa1a44615e 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -415,10 +415,9 @@ def _unset_reason(view: _View) -> str: ) -# The fingerprint fields a CLI's gate judges: the stores it would write. A CLI verifies -# no token and its fingerprint never reaches the router, so the rest of the servers' -# fingerprint is not its to match (docs/architecture/cells.md, "The CLI declares its -# orgs"). +# An org-declaring CLI's gate checks the stores it would write: it verifies no token +# and sends no fingerprint to the router. Operator placement checks the full fingerprint +# separately (docs/architecture/cells.md, "The CLI declares its orgs"). _STORE_FIELDS = ("deployment_id", "bucket", "bucket_prefix") diff --git a/server/router/app.py b/server/router/app.py index 4e24a7c922..7c80df19af 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -14,6 +14,8 @@ - `POST /cells/v1/operator/cells//limit` and `.../credential`: the operator's changes to a cell's row (`loom cell limit`, `loom cell credential`), with the router's operator credential; +- `POST /cells/v1/operator/cells//place`: `loom cell place` places an org with the + operator credential, verifying the requested cell's deployment id and fingerprint; - `POST /cells/v1/operator/cells//register`: `loom cell prepare` registers a cell, once the probe listener (the first cell's at LOOM_ROUTER_UPSTREAM_PROBE_URL, a later cell's at its `probe_url`) answers its deployment id, prepare's nonces and the bucket and @@ -28,9 +30,9 @@ Every refusal of a placement writes nothing. A malformed placement body answers 400; the directory's refusals answer 503, so the cell's request answers 503 and its retry is safe, except a new org longer than `cell.ORG_MAX_BYTES`, which answers 400 `ORG_TOO_LONG` and no -retry places; a credential that names no cell, or not this one, answers 401. An operator -route answers a cell the directory does not hold 404 with the code `NO_SUCH_CELL`, so a 404 -without it is a router of an earlier build. +retry places; a credential that names no cell, or not this one, answers 401. Operator +placement refusals answer 409; the other operator routes answer a cell the directory does +not hold 404 with `NO_SUCH_CELL`, so a 404 without it is a router of an earlier build. """ from __future__ import annotations @@ -301,7 +303,7 @@ def refusal(locked: directory.Cell) -> str: return "" try: - winner, deployment_id = await directory.place(org, name, refusal) + winner, deployment_id = await directory.place(org, name, refusal, actor=f"cell:{name}") except directory.OrgTooLong as e: return _refused(400, e.reason, code="ORG_TOO_LONG") except (directory.PlacementRefused, directory.DirectoryUnavailable) as e: @@ -314,6 +316,41 @@ def refusal(locked: directory.Cell) -> str: log.info("placement of %s: %s (asked by %s)", org, winner, name) return JSONResponse({"cell": winner, "deployment_id": deployment_id}) + async def operator_place(self, request: Request) -> JSONResponse: + """Place with the operator credential. The CLI checks its local Postgres mode; + this route trusts that check and verifies the directory's locked identity.""" + body = await self._operator_body(request) + if isinstance(body, JSONResponse): + return body + org = cell_state.named_org(body.get("org")) + deployment_id = _text(body, "deployment_id", needed=True) + if ( + set(body) != {"org", "deployment_id", "fingerprint"} + or org is None + or deployment_id is None + ): + return _refused(400, "a placement names org, deployment_id and fingerprint") + if reason := cell_state.org_name_refusal(org): + return _refused(400, reason, code="ORG_TOO_LONG") + name = request.path_params["cell"] + + def verify(locked: directory.Cell) -> str: + if locked.deployment_id != deployment_id: + return "the calling process names another deployment id" + if not fingerprint_matches(locked, body["fingerprint"]): + return f"the calling process's configuration matches no value of {name!r}" + return "" + + try: + winner, deployment = await directory.place( + org, name, placement_refusal, actor="operator", verify=verify + ) + except directory.PlacementRefused as e: + return _refused(409, e.reason) + except directory.DirectoryUnavailable as e: + return _refused(503, e.reason, code="BACKEND_UNAVAILABLE") + return JSONResponse({"org": org, "cell": winner, "deployment_id": deployment}) + async def cell_state(self, request: Request) -> JSONResponse: """`GET /cells/v1/cells/`: the first cell's state, for its own credential only, from this replica's last read of the row (`_state_row`). A first cell in @@ -1365,6 +1402,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: return Starlette( routes=[ Route("/cells/v1/placements", router.place, methods=["POST"]), + Route("/cells/v1/operator/cells/{cell}/place", router.operator_place, methods=["POST"]), Route("/cells/v1/cells/{cell}", router.cell_state, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/limit", router.set_limit, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}/register", router.register, methods=["POST"]), diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..52893ef4ef 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -12,7 +12,8 @@ placing one org at once agree on the winner, and a retry of the same call answers the same; a batch takes none, since no other cell exists while it runs; - the placements' primary key, as a backstop: every insert is `ON CONFLICT DO NOTHING`, and - only a row it returns is counted and audited. + only a row it returns is counted. Every successful call writes an audit row before + committing, so a duplicate winner waits for synchronous durability too. """ from __future__ import annotations @@ -59,6 +60,10 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) +# New placement: BEGIN, org lock, winner, cell lock, winner, insert, counter, winner, +# audit, COMMIT. The audit also gives duplicate winners a synchronous durability fence. +PLACE_STATEMENTS = 10 +PLACE_MAX_S = _bound_s(PLACE_STATEMENTS) # not a setting: derived # The longest a limit or credential change can take: eight statements (begin, lock, row, # check, update, audit, read-back, commit). Registration has its own bound, below. OPERATOR_CHANGE_MAX_S = _bound_s(8) # not a setting: derived from the bound above @@ -635,39 +640,49 @@ async def first_cell_named(cell_name: str) -> Cell | None: return _cell(row) if row else None -async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: - """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) - that holds it. An org placed already answers its cell, whatever the cell's state. - Otherwise the cell's row is locked, the winner read again (an attach batch holds the row - lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it - waits for the placement and the quota holds across router replicas; a refused placement - raises PlacementRefused and writes nothing, and a new org name longer than - `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as - any placed org does).""" +async def place( + org: str, + cell_name: str, + refusal: Callable[[Cell], str], + *, + actor: str, + verify: Callable[[Cell], str] | None = None, +) -> tuple[str, str]: + """The permanent winner of `org`, with every successful call audited and committed + synchronously. `verify` judges the requested cell's locked identity even for a winner + already there; `refusal` and quota judge new placements only. A winner elsewhere needs + neither check. The org lock and cell row lock serialize live and attach placements.""" async with _reading(), _connection() as conn, conn.transaction(): - # Placements of one org run one at a time, whichever cell asks: the later one - # reads the earlier one's winner. await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) row = await conn.fetchrow(_WINNER, org) - if row is None: - _refuse_long([org]) + if row is None or (verify is not None and row["cell"] == cell_name): + if row is None: + _refuse_long([org]) locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) if locked is None: raise PlacementRefused(f"cell {cell_name!r} is not in the directory") - # An attach batch of this cell, which holds the row lock and not the org's, may - # have placed the org since the first read: it is answered as placed. + candidate = _cell(locked) + # An attach batch holds this row's lock rather than the org's. row = await conn.fetchrow(_WINNER, org) + if (row is None or row["cell"] == cell_name) and verify is not None: + if reason := verify(candidate): + raise PlacementRefused(reason) if row is None: - await _place_one(conn, org, cell_name, _cell(locked), refusal) + await _place_one(conn, org, cell_name, candidate, refusal) row = await conn.fetchrow(_WINNER, org) - return str(row["cell"]), str(row["deployment_id"]) + winner = str(row["cell"]), str(row["deployment_id"]) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + actor, + {"org": org, "cell": winner[0]}, + ) + return winner async def _place_one( conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] ) -> None: - """Judge the locked row (`refusal`, then its quota) and insert the placement; its count - and audit row only when the insert wrote it.""" + """Judge the locked row and insert the placement, counted only when written.""" if reason := refusal(candidate): raise PlacementRefused(reason) quota = candidate.placements_max @@ -684,11 +699,6 @@ async def _place_one( await conn.execute( "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name ) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", - f"cell:{cell_name}", - {"org": org, "cell": cell_name}, - ) def _refuse_long(orgs: Iterable[str]) -> None: diff --git a/tests/integration/test_cell_place.py b/tests/integration/test_cell_place.py new file mode 100644 index 0000000000..7024024002 --- /dev/null +++ b/tests/integration/test_cell_place.py @@ -0,0 +1,238 @@ +"""Operator placement on two cells' databases and a real directory, without memory rows.""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import os +import time + +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.router import app, directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database, own_database + +OPERATOR = "lco_" + "3" * 128 +ROUTER = httpx.URL("http://127.0.0.1:7901") + + +@pytest.fixture +async def stores(monkeypatch): + admin = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, dsn = await create_database(admin) + async with contextlib.AsyncExitStack() as stack: + stack.push_async_callback(drop_database, admin, name) + first = await stack.enter_async_context(own_database()) + second = await stack.enter_async_context(own_database()) + monkeypatch.setattr(directory, "_pool", None) + stack.push_async_callback(directory.close) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + await directory.migrate() + pg.expect_deployment(None) + router_app = app.build_app(app.Router()) + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kw: real_client(transport=httpx.ASGITransport(app=router_app), **kw), + ) + targets = {"first": first, "second": second} + try: + for cell_name, cell_dsn in targets.items(): + await _initialize(cell_dsn, cell_name) + yield targets + finally: + pg.expect_deployment(None) + await pg.close_pool() + await directory.close() + + +async def _initialize(dsn, name): + import asyncpg + + fields = cell.fingerprint(f"dep-{name}") + conn = await asyncpg.connect(dsn) + try: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + f"dep-{name}", + ) + await conn.execute( + "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", name == "first", name + ) + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6)", + name, + name == "first", + f"{name}.example.test", + hashlib.sha256(name.encode()).hexdigest(), + f"dep-{name}", + cell.as_value(fields), + ) + + +async def _local(stores, monkeypatch, name, org="acme"): + monkeypatch.setattr(settings, "cell", name) + monkeypatch.setattr(settings, "pg_dsn", stores[name]) + return await cell_place.place(argparse.Namespace(cell=name, org=org), ROUTER) + + +async def _tenancy(dsn): + import asyncpg + + conn = await asyncpg.connect(dsn) + try: + return [r["org"] for r in await conn.fetch("SELECT org FROM loom.tenancy ORDER BY org")] + finally: + await conn.close() + + +async def test_operator_place_retries_and_refuses_a_different_cell( + stores, monkeypatch, record_property +): + started = time.perf_counter() + first = await _local(stores, monkeypatch, "first") + record_property("local_first_place_ms", (time.perf_counter() - started) * 1000) + assert first == {"org": "acme", "cell": "first", "deployment_id": "dep-first"} + started = time.perf_counter() + assert await _local(stores, monkeypatch, "first") == first + record_property("local_duplicate_place_ms", (time.perf_counter() - started) * 1000) + assert await _tenancy(stores["first"]) == ["acme"] + with pytest.raises(cell_place.PlacementRefused, match="belongs to cell 'first'"): + await _local(stores, monkeypatch, "second") + assert await _tenancy(stores["second"]) == [] + async with directory._connection() as conn: + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'first'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 3 + ) + + +async def test_two_cells_racing_one_org_have_one_winner_and_count_once(stores): + async def place(name): + fields = cell.as_value(cell.fingerprint(f"dep-{name}")) + async with httpx.AsyncClient() as client: + return await client.post( + str(ROUTER.copy_with(path=f"/cells/v1/operator/cells/{name}/place")), + json={"org": "raced", "deployment_id": f"dep-{name}", "fingerprint": fields}, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + a, b = await asyncio.gather(place("first"), place("second")) + assert a.status_code == b.status_code == 200 + assert a.json() == b.json() + async with directory._connection() as conn: + assert await conn.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert await _tenancy(stores["first"]) == await _tenancy(stores["second"]) == [] + + +@pytest.mark.parametrize("limit", ["closed = true", "placements_max = 0"]) +async def test_new_closed_or_full_cell_writes_neither_store(stores, monkeypatch, limit): + async with directory._connection() as conn: + await conn.execute(f"UPDATE loom_router.cells SET {limit} WHERE cell = 'first'") + with pytest.raises(cell_place.PlacementRefused, match="router refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert await directory.lookup("acme") is None + + +async def test_failed_local_insert_resumes_and_existing_winner_ignores_closed_quota( + stores, monkeypatch +): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute( + "CREATE FUNCTION loom.fail_place() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'local insertion refused'; END $$" + ) + await conn.execute( + "CREATE TRIGGER fail_place BEFORE INSERT ON loom.tenancy FOR EACH ROW EXECUTE FUNCTION loom.fail_place()" + ) + with pytest.raises(asyncpg.RaiseError, match="local insertion refused"): + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == [] + assert (await directory.lookup("acme"))[0] == "first" + await conn.execute("DROP TRIGGER fail_place ON loom.tenancy") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +async def test_first_recording_refuses_and_enforced_attaching_succeeds(stores, monkeypatch): + import asyncpg + + conn = await asyncpg.connect(stores["first"]) + try: + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + with pytest.raises(cell_place.PlacementRefused, match="needs enforced"): + await _local(stores, monkeypatch, "first") + assert await directory.lookup("acme") is None + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") + async with directory._connection() as dc: + await dc.execute( + "UPDATE loom_router.cells SET state = 'attaching' WHERE cell = 'first'" + ) + await _local(stores, monkeypatch, "first") + assert await _tenancy(stores["first"]) == ["acme"] + finally: + await conn.close() + + +@pytest.mark.parametrize("changed", ["mode", "fingerprint"]) +async def test_changed_local_state_after_directory_call_refuses_insert( + stores, monkeypatch, changed +): + import asyncpg + + real = cell_place._directory + + async def change_after_call(*args): + await real(*args) + conn = await asyncpg.connect(stores["first"]) + try: + if changed == "mode": + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") + else: + fields = cell.fingerprint("dep-first") + fields["oidc_client_id"] = "changed" + await conn.execute( + "UPDATE loom.cell_self SET current_fp = $1::jsonb", + json.dumps(cell.as_value(fields)), + ) + finally: + await conn.close() + + monkeypatch.setattr(cell_place, "_directory", change_after_call) + with pytest.raises(cell_place.PlacementRefused): + await _local(stores, monkeypatch, "first") + assert (await directory.lookup("acme"))[0] == "first" + assert await _tenancy(stores["first"]) == [] diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index caf092e9ec..94b25a95a9 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -109,7 +109,8 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, org ) racing = asyncio.gather( - directory.place(org, a, _never), directory.place(org, b, _never) + directory.place(org, a, _never, actor="operator"), + directory.place(org, b, _never, actor="operator"), ) # Both placements wait on the org's lock before it is released. for _ in range(100): @@ -133,10 +134,11 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: "SELECT actor FROM loom_router.audit WHERE action = 'place' AND detail->>'org' = $1", org, ) - assert [r["actor"] for r in audit] == [f"cell:{winners[0][0]}"] + assert [r["actor"] for r in audit] == ["operator", "operator"] # Ten concurrent placements in a cell with room for three: three are written. results = await asyncio.gather( - *(directory.place(f"{q}-{i}", q, _never) for i in range(10)), return_exceptions=True + *(directory.place(f"{q}-{i}", q, _never, actor="operator") for i in range(10)), + return_exceptions=True, ) assert sum(isinstance(r, tuple) for r in results) == 3 assert sum(isinstance(r, directory.PlacementRefused) for r in results) == 7 @@ -145,10 +147,13 @@ async def test_the_directory_places_once_and_holds_the_quota(dsn: str) -> None: assert held == 3 # An org placed already answers its cell, whatever the quota or the cell's state. placed = next(f"{q}-{i}" for i, r in enumerate(results) if isinstance(r, tuple)) - assert await directory.place(placed, q, lambda _r: "closed") == (q, f"dep-{q}") + assert await directory.place(placed, q, lambda _r: "closed", actor="operator") == ( + q, + f"dep-{q}", + ) # A refusal judged on the locked row writes nothing. with pytest.raises(directory.PlacementRefused, match="closed"): - await directory.place(f"new-{a}", a, lambda _r: "closed") + await directory.place(f"new-{a}", a, lambda _r: "closed", actor="operator") assert await directory.lookup(f"new-{a}") is None # Routing reads every cell with its upstream address ('' until attach writes it). cells, _aliases = await directory.routing() @@ -738,7 +743,9 @@ async def _held(*a: Any) -> None: conn = await asyncpg.connect(own_dsn) try: async with asyncio.timeout(30): - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await inserted.wait() batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme", "beta"])) await _waits_on_the_row_lock(conn, "FOR UPDATE") @@ -782,12 +789,17 @@ async def _held(cell_name: str) -> Any: async with asyncio.timeout(30): batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme"])) await written.wait() - live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) + live = asyncio.ensure_future( + directory.place("acme", name, lambda _row: "", actor=f"cell:{name}") + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await batch == (1, 1) assert await live == (name, dep) - assert await _placements_of(conn, name) == (1, [("operator", "acme")]) + assert await _placements_of(conn, name) == ( + 1, + [("operator", "acme"), (f"cell:{name}", "acme")], + ) finally: release.set() await conn.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 4e45db4285..40ef634062 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", - "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", + "usage: loom cell [-h] {credential,limit,which,remove,place,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,place,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -455,6 +455,10 @@ "usage: loom cell remove [-h] --cell CELL [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL The cell to remove (its LOOM_CELL) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell place": [ + "usage: loom cell place [-h] --cell CELL --org ORG [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --org ORG The org to place --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom cell prepare": [ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index a31a862106..37cebe51bb 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12811,6 +12811,99 @@ } ] }, + "place": { + "prog": "loom cell place", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_place:cmd_place" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "org", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--org" + ], + "dest": "org", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The org to place", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] + }, "prepare": { "prog": "loom cell prepare", "usage": null, @@ -13163,6 +13256,7 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", + "place": "loom cell place", "prepare": "loom cell prepare", "attach": "loom cell attach" }, @@ -13187,6 +13281,11 @@ "help": "Remove the directory row of a cell attaching with no placement.", "metavar": "remove" }, + { + "dest": "place", + "help": "Place an org in this cell and record its tenancy.", + "metavar": "place" + }, { "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py new file mode 100644 index 0000000000..5f8deeaf0e --- /dev/null +++ b/tests/unit/test_cell_place.py @@ -0,0 +1,466 @@ +"""Operator placement keeps the directory winner and bounds its local session.""" + +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import asynccontextmanager +from dataclasses import asdict +from typing import Any + +import pytest +from starlette.testclient import TestClient + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.unit.test_router import _row + +OPERATOR = "lco_" + "7" * 128 + + +@pytest.fixture(autouse=True) +def _code_org_reader(monkeypatch): + monkeypatch.setattr(cell, "_code_orgs", None) + + +class DirectoryConnection: + def __init__(self, winner: str | None = "eu-1") -> None: + self.winner = winner + self.row = _row() + self.calls: list[tuple[str, tuple[Any, ...]]] = [] + self.committed = False + + @asynccontextmanager + async def transaction(self): + yield + self.committed = True + + async def execute(self, sql, *args): + self.calls.append((sql, args)) + + async def fetchrow(self, sql, *args): + self.calls.append((sql, args)) + if "FROM loom_router.cells WHERE" in sql: + return asdict(self.row) + if self.winner is not None: + return {"cell": self.winner, "deployment_id": f"dep-{self.winner}"} + return None + + async def fetchval(self, sql, *args): + self.calls.append((sql, args)) + self.winner = args[1] + return args[0] + + +@pytest.fixture +def connected(monkeypatch): + conn = DirectoryConnection() + + @asynccontextmanager + async def connection(**kw): + yield conn + + monkeypatch.setattr(directory, "_connection", connection) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() + ) + return conn + + +def request(client, **over): + return client.post( + "/cells/v1/operator/cells/eu-1/place", + json={ + "org": "acme", + "deployment_id": "dep-eu-1", + "fingerprint": cell.as_value( + {"public_host": "eu-1.example.test", "deployment_id": "dep-eu-1"} + ), + **over, + }, + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + + +def test_existing_winner_emits_audit_wal_before_returning(connected): + client = TestClient(app.build_app(app.Router())) + result = request(client) + assert result.status_code == 200 + assert result.json() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + audit = [(sql, args) for sql, args in connected.calls if "INSERT INTO loom_router.audit" in sql] + assert len(audit) == 1 and audit[0][1] == ("operator", {"org": "acme", "cell": "eu-1"}) + assert connected.committed + assert not any("UPDATE loom_router.cells SET placed" in sql for sql, _ in connected.calls) + + +@pytest.mark.parametrize( + "posted", + [{"deployment_id": "other"}, {"fingerprint": {}}, {"org": "x" * (cell.ORG_MAX_BYTES + 1)}], +) +def test_existing_winner_refuses_operator_identity_mismatch(connected, posted): + result = request(TestClient(app.build_app(app.Router())), **posted) + assert result.status_code in (400, 409) + assert not connected.committed + assert not any("INSERT" in sql for sql, _ in connected.calls) + + +def test_operator_authentication_precedes_directory_access(connected): + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", json={} + ) + assert result.status_code == 401 and connected.calls == [] + + +def test_live_duplicate_also_emits_audit_wal(connected): + assert asyncio.run( + directory.place("acme", "eu-1", lambda row: "closed", actor="cell:eu-1") + ) == ("eu-1", "dep-eu-1") + assert connected.committed + assert any("INSERT INTO loom_router.audit" in sql for sql, _ in connected.calls) + + +class LocalConnection: + def __init__(self): + self.calls = [] + self.committed = False + self.terminated = False + self.stall_commit = False + self.stall_close = False + self.fail_insert = False + self.commit_started = asyncio.Event() + + def transaction(self): + conn = self + + class Transaction: + async def start(self): + pass + + async def commit(self): + conn.commit_started.set() + if conn.stall_commit: + await asyncio.Event().wait() + conn.committed = True + + return Transaction() + + async def execute(self, sql, *args): + self.calls.append(sql) + if "INSERT" in sql and self.fail_insert: + raise OSError("local insert failed") + return "INSERT 0 0" + + def terminate(self): + self.terminated = True + + +@pytest.fixture +def local(monkeypatch): + from server.cli import cell_place + from server.core import pg + + conn = LocalConnection() + state = {"first": "eu-1", "state": "enforced", "deployment": "dep-eu-1", "row": None} + + async def read_state(c): + return state["state"], state["first"] + + async def fetchval(sql): + return state["deployment"] + + async def fetchrow(sql): + return state["row"] + + conn.fetchval = fetchval + conn.fetchrow = fetchrow + + @asynccontextmanager + async def dedicated(**kw): + try: + yield conn + finally: + if conn.stall_close: + await asyncio.Event().wait() + + async def direct(c, what): + return [] + + async def take_lock(c, what): + conn.calls.append("lock") + + async def held(c): + return True + + async def directory_call(*a): + conn.calls.append("directory") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell, "read_state_if_present", read_state) + monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) + monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) + monkeypatch.setattr(cell_place.apg, "lock_held", held) + monkeypatch.setattr(cell_place, "_directory", directory_call) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.03) + fields = cell.fingerprint("dep-eu-1") + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + return conn, state + + +async def invoke_place(): + import argparse + + import httpx + + from server.cli import cell_place + + return await cell_place.place( + argparse.Namespace(cell="eu-1", org="acme"), httpx.URL("https://router.test") + ) + + +async def test_local_noop_insert_still_commits_durability_message(local): + from server.core import pg + + conn, state = local + assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + assert conn.committed and conn.terminated + assert conn.calls[0:2] == ["lock", "directory"] + assert conn.calls[-1] == pg._DURABLE_MESSAGE + + +@pytest.mark.parametrize( + "change", + [ + {"state": "recording"}, + {"state": "partial"}, + {"first": "us-1"}, + {"deployment": "other"}, + {"row": None}, + ], +) +async def test_local_invalid_identity_refuses_before_directory(local, change): + from server.cli import cell_place + + conn, state = local + state.update(change) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert conn.calls == ["lock"] and conn.terminated + + +@pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) +async def test_local_identity_is_read_again_before_insert(local, monkeypatch, change): + from server.cli import cell_place + + conn, state = local + + async def changed(*a): + state.update(change) + + monkeypatch.setattr(cell_place, "_directory", changed) + with pytest.raises(cell_place.PlacementRefused): + await invoke_place() + assert not any("INSERT" in sql for sql in conn.calls) + assert conn.terminated and not conn.committed + + +async def test_local_insert_failure_can_be_retried(local): + conn, state = local + conn.fail_insert = True + with pytest.raises(OSError, match="local insert failed"): + await invoke_place() + assert not conn.committed and conn.terminated + conn.fail_insert = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_and_cleanup_stalls_are_bounded(local): + conn, state = local + conn.stall_commit = conn.stall_close = True + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert conn.commit_started.is_set() and conn.terminated and not conn.committed + conn.stall_commit = conn.stall_close = False + await invoke_place() + assert conn.committed + + +async def test_local_commit_cancellation_closes_the_session(local): + conn, state = local + conn.stall_commit = True + task = asyncio.create_task(invoke_place()) + await conn.commit_started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert conn.terminated and not conn.committed + + +async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): + from server.core import pg + + conn, state = local + cleaned = False + + @asynccontextmanager + async def stalled(**kw): + nonlocal cleaned + try: + await asyncio.Event().wait() + yield conn + finally: + cleaned = True + + monkeypatch.setattr(pg, "dedicated", stalled) + with pytest.raises(TimeoutError): + await asyncio.wait_for(invoke_place(), 0.5) + assert cleaned and conn.calls == [] + + +async def test_query_timeout_cannot_enter_an_unbounded_rollback(local): + conn, state = local + + class Transaction: + async def start(self): + pass + + async def commit(self): + pytest.fail("a stalled statement cannot commit") + + async def __aenter__(self): + await self.start() + + async def __aexit__(self, *exc): + await asyncio.Event().wait() + + async def stalled(sql, *args): + await asyncio.Event().wait() + + conn.transaction = Transaction + conn.execute = stalled + task = asyncio.create_task(invoke_place()) + try: + done, _ = await asyncio.wait([task], timeout=0.15) + assert done, "the statement deadline must include ending its transaction" + with pytest.raises(TimeoutError): + await task + assert conn.terminated + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize( + "status,answer", + [ + (200, {"org": "acme", "cell": "us-1", "deployment_id": "dep-us-1"}), + (200, {"org": "acme", "cell": "eu-1", "deployment_id": "wrong"}), + (200, []), + (200, {"org": "someone-else", "cell": "eu-1", "deployment_id": "dep-eu-1"}), + (503, {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"}), + (504, {"error": "gateway timeout"}), + ], +) +async def test_cli_refuses_unknown_response_or_other_winner(monkeypatch, status, answer): + import httpx + + from server.cli import cell_place + + async def respond(request): + assert request.headers["authorization"] == f"Bearer {settings.router_operator_token}" + return httpx.Response(status, json=answer) + + real = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real(transport=httpx.MockTransport(respond), **kw) + ) + with pytest.raises(cell_place.PlacementRefused): + await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +async def test_repository_only_configured_orgs_are_in_full_cli_fingerprint(local, monkeypatch): + from server.cli import cell_place + from server.code.domain.index import repos + + conn, state = local + monkeypatch.setattr(repos, "operator_orgs", lambda: {"repo-only"}) + fields = cell.fingerprint("dep-eu-1") + fields["configured_orgs"] = sorted(set(fields["configured_orgs"]) | {"repo-only"}) + state["row"] = {"current_fp": cell.as_value(fields), "next_fp": None} + _, value = await cell_place._identity(conn, "eu-1") + assert "repo-only" in value["fields"]["configured_orgs"] + + +def test_new_placement_uses_counted_statement_bound(monkeypatch): + from tests.unit.test_router_attach import _cell_row, _Conn, _Store + + store = _Store(_cell_row(state="serving")) + + @asynccontextmanager + async def connection(**kw): + yield _Conn(store) + + monkeypatch.setattr(directory, "_connection", connection) + asyncio.run( + directory.place("new", "eu-1", lambda row: "", actor="operator", verify=lambda row: "") + ) + assert store.statements == directory.PLACE_STATEMENTS + assert directory.PLACE_MAX_S == directory._bound_s(store.statements) + + +def test_operator_placement_body_bound_refuses_before_directory(connected): + from server.router.limits import MAX_BODY_BYTES + + result = TestClient(app.build_app(app.Router())).post( + "/cells/v1/operator/cells/eu-1/place", + content=b"x" * (MAX_BODY_BYTES + 1), + headers={"Authorization": f"Bearer {OPERATOR}"}, + ) + assert result.status_code == 413 and connected.calls == [] + + +def test_operator_place_bypasses_cell_token_rate_limiter(connected, monkeypatch): + router = app.Router() + monkeypatch.setattr( + router.placements, "admit", lambda name: pytest.fail("operator used cell rate") + ) + assert request(TestClient(app.build_app(router))).status_code == 200 + + +@pytest.mark.parametrize("org", ["", "-", "*", "bad\x00org"]) +def test_operator_place_refuses_malformed_org_before_directory(connected, org): + assert request(TestClient(app.build_app(app.Router())), org=org).status_code == 400 + assert connected.calls == [] + + +def test_cli_cancellation_names_the_idempotent_retry_and_preserves_cancellation( + monkeypatch, capsys +): + import argparse + + import httpx + + from server.cli import cell_place + + cancelled = asyncio.CancelledError("commit response was cancelled") + + async def cancel(*args): + raise cancelled + + monkeypatch.setattr(cell_place, "_args_problem", lambda args: "") + monkeypatch.setattr( + cell_place.cell_cli, "operator_router", lambda args: httpx.URL("https://router.test") + ) + monkeypatch.setattr(cell_place, "place", cancel) + monkeypatch.setattr(cell_place, "run_connected", lambda run, **kw: asyncio.run(run(None))) + with pytest.raises(asyncio.CancelledError) as caught: + cell_place.cmd_place(argparse.Namespace(cell="eu-1", org="acme")) + assert caught.value is cancelled + printed = capsys.readouterr() + assert printed.out == "" + assert "placement may have committed; rerun the same command" in printed.err diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 9de51a1e75..ff59af4e14 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -82,7 +82,9 @@ async def first_cell_named(self, name: str) -> directory.Cell | None: row = self.cells.get(name) return row if row is not None and row.is_first else None - async def place(self, org: str, name: str, refusal: Any) -> tuple[str, str]: + async def place( + self, org: str, name: str, refusal: Any, *, actor: str, verify: Any = None + ) -> tuple[str, str]: self._check() if org not in self.placements: row = self.cells[name] # the row as the placement locks it diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index efb0b29e80..bab0420cc4 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -900,7 +900,7 @@ def sent(call: Any) -> int: def test_a_live_placement_reads_the_winner_again_under_the_row_lock(store: _Store) -> None: """A batch holds the cell's row lock, not the org's: an org it placed between the live placement's first read and its row lock is answered placed, never judged on the quota, - counted or audited twice.""" + counted twice. Its call still writes the durability audit.""" store.cells["eu-1"].update(state="serving", placements_max=0) reads = 0 real = _Conn.fetchrow @@ -915,9 +915,10 @@ async def _fetchrow(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchrow", _fetchrow) - winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) + winner = asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) assert winner == ("eu-1", "dep-1") and reads == 2 - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] def test_a_live_insert_that_loses_to_the_primary_key_counts_nothing(store: _Store) -> None: @@ -931,8 +932,12 @@ async def _fetchval(self: _Conn, sql: str, *a: Any) -> Any: with pytest.MonkeyPatch.context() as mp: mp.setattr(_Conn, "fetchval", _fetchval) - assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "")) == ("eu-1", "dep-1") - assert store.cells["eu-1"]["placed"] == 0 and store.audit == [] + assert asyncio.run(directory.place("acme", "eu-1", lambda _row: "", actor="cell:eu-1")) == ( + "eu-1", + "dep-1", + ) + assert store.cells["eu-1"]["placed"] == 0 + assert store.audit == [("cell:eu-1", "place", {"org": "acme", "cell": "eu-1"})] # The attach lock and the troubleshooting page. From 042dd5e42c27680d8bc9c198101c71dc1733191e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:41:50 +0000 Subject: [PATCH 017/181] Prove cell placement retries survive standby promotion Exercise primary-visible commits, durable duplicate retries and cancellation cleanup on an isolated physical standby pair in both supported Postgres CI jobs. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ .../integration/test_cell_place_durability.py | 310 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 tests/integration/test_cell_place_durability.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..702540d1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -899,7 +899,36 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_place.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest placement durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-place-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_CELL_PLACE_DRILL: "1" + LOOM_CELL_PLACE_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-place-drill-pg loom-scale-place-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-place-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-place-drill-pg loom-scale-place-drill-pg-standby || true + docker network rm loom-scale-place-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby + pytest -q tests/integration/test_cell_place_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/tests/integration/test_cell_place_durability.py b/tests/integration/test_cell_place_durability.py new file mode 100644 index 0000000000..1d788f47d6 --- /dev/null +++ b/tests/integration/test_cell_place_durability.py @@ -0,0 +1,310 @@ +"""Placement retries survive promotion of a private physical standby. + +Only the dedicated CI drill enables this file. Its pair is separate from the +integration server's stores because it pauses replication and stops the primary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import json +import os +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlsplit + +import asyncpg +import httpx +import pytest + +from server.cli import cell_place +from server.core import cell, pg +from server.core.schema import pg_migrate +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-place-drill-pg" +STANDBY = PRIMARY + "-standby" +ORG = "placement-durability" +DEPLOYMENT = "placement-drill" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +@dataclass +class Drill: + observer: Any + directory_dsn: str + local_dsn: str + tasks: list[asyncio.Task] = field(default_factory=list) + paused: bool = False + + def task(self, work: Any) -> asyncio.Task: + task = asyncio.create_task(work) + self.tasks.append(task) + return task + + async def pause(self) -> None: + await _docker("pause", STANDBY) + self.paused = True + + async def resume(self) -> None: + await _docker("unpause", STANDBY) + self.paused = False + + async def waiting(self, task: asyncio.Task, application: str) -> int: + async with asyncio.timeout(5): + while True: + assert not task.done(), "placement acknowledged before the standby resumed" + pid = await self.observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + application, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.02) + + async def seed(self, dsn: str, sql: str, *, marked: bool = False) -> None: + """Cancel COMMIT only after it is primary-visible and waiting for standby flush.""" + application = f"placement-seed-{len(self.tasks)}" + conn = await asyncpg.connect( + dsn, + timeout=5, + command_timeout=5, + server_settings={ + "application_name": application, + "synchronous_commit": "on", + **(pg.marker_settings() if marked else {}), + }, + ) + try: + await conn.execute("BEGIN") + await conn.execute(sql) + commit = self.task(conn.execute("COMMIT")) + await self.waiting(commit, application) + commit.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(commit, 5) + finally: + conn.terminate() + + +@pytest.fixture +async def drill(monkeypatch): + if not os.environ.get("LOOM_CELL_PLACE_DRILL"): + pytest.skip("LOOM_CELL_PLACE_DRILL not set: requires its disposable physical standby") + assert os.environ["LOOM_CELL_PLACE_DRILL"] == "1" + dsn = os.environ["LOOM_CELL_PLACE_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + state = None + try: + # Refuse to disrupt a container unless the explicit DSN identifies this exact pair. + identifier = str( + await observer.fetchval("SELECT system_identifier FROM pg_control_system()") + ) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + _, directory_dsn = await create_database(dsn) + _, local_dsn = await create_database(dsn) + state = Drill(observer, directory_dsn, local_dsn) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(settings, "router_pg_dsn", directory_dsn) + monkeypatch.setattr(settings, "pg_dsn", local_dsn) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_statement_timeout_ms", 10_000) + previous_expected = pg._expected + previous_orgs = cell._code_orgs + pg.expect_deployment(None) + from server.code.domain.index import repos + + cell.set_code_orgs(repos.operator_orgs) + try: + async with asyncio.timeout(60): + await _initialize(state) + yield state + finally: + pg.expect_deployment(previous_expected) + cell._code_orgs = previous_orgs + finally: + if state is not None: + if state.paused: + await state.resume() + for task in state.tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(15): + await asyncio.gather(*state.tasks, return_exceptions=True) + await directory.close() + await pg.close_pool() + observer.terminate() + + +async def _initialize(drill: Drill) -> None: + @contextlib.asynccontextmanager + async def connect(): + conn = await asyncpg.connect(drill.local_dsn, command_timeout=10) + try: + yield conn + finally: + conn.terminate() + + await pg_migrate.migrate(connect=connect) + await directory.migrate() + fingerprint = cell.as_value(cell.fingerprint(DEPLOYMENT)) + async with connect() as conn: + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, $1)", + DEPLOYMENT, + ) + await conn.execute("INSERT INTO loom.cell_first (is_first, cell) VALUES (true, 'first')") + await conn.execute("INSERT INTO loom.tenancy_mode (mode) VALUES ('enforced')") + await conn.execute( + "INSERT INTO loom.cell_self (current_fp) VALUES ($1::jsonb)", json.dumps(fingerprint) + ) + async with directory._connection() as conn: + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, state, mode, credential_sha256, deployment_id, current_fp) " + "VALUES ('first', true, 'first.example.test', 'serving', 'enforced', $1, $2, $3)", + "3" * 64, + DEPLOYMENT, + fingerprint, + ) + + +async def _directory_retry(drill: Drill) -> None: + await drill.pause() + await drill.seed( + drill.directory_dsn, + "INSERT INTO loom_router.placements (org, cell) VALUES ('placement-durability', 'first'); " + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = 'first'", + ) + assert await directory.lookup(ORG) == ("first", "first.example.test") + retry = drill.task(directory.place(ORG, "first", lambda _: "", actor="operator")) + await drill.waiting(retry, "loom-router") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == ("first", DEPLOYMENT) + + +async def _local_retry(drill: Drill, monkeypatch) -> None: + async def verified_directory(router, name, org, deployment, fingerprint): + # The directory retry above already acknowledged this winner's physical flush. + assert (name, org, deployment) == ("first", ORG, DEPLOYMENT) + assert await directory.lookup(org) == (name, "first.example.test") + + monkeypatch.setattr(cell_place, "_directory", verified_directory) + await drill.pause() + await drill.seed( + drill.local_dsn, + "INSERT INTO loom.tenancy (org) VALUES ('placement-durability')", + marked=True, + ) + assert ( + await _sql( + PRIMARY, + urlsplit(drill.local_dsn).path.removeprefix("/"), + "SELECT org FROM loom.tenancy", + ) + == ORG + ) + args = argparse.Namespace(cell="first", org=ORG) + cancelled = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + pid = await drill.waiting(cancelled, "loom") + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cancelled, cell_place._bound_s() + 1) + async with asyncio.timeout(5): + while True: + if not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.02) + assert not await drill.observer.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ) + retry = drill.task(cell_place.place(args, httpx.URL("http://router.example.test"))) + await drill.waiting(retry, "loom") + await drill.resume() + assert await asyncio.wait_for(retry, 5) == { + "org": ORG, + "cell": "first", + "deployment_id": DEPLOYMENT, + } + + +@pytest.mark.recovery +@pytest.mark.destructive +async def test_duplicate_placement_survives_physical_standby_promotion(drill, monkeypatch): + async with asyncio.timeout(120): + await _directory_retry(drill) + await _local_retry(drill, monkeypatch) + await directory.close() + await pg.close_pool() + drill.observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + directory_db = urlsplit(drill.directory_dsn).path.removeprefix("/") + local_db = urlsplit(drill.local_dsn).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT p.org, p.cell, c.deployment_id, c.placed FROM loom_router.placements p " + "JOIN loom_router.cells c USING (cell)", + ) + == f"{ORG}|first|{DEPLOYMENT}|1" + ) + assert ( + await _sql( + STANDBY, + directory_db, + "SELECT actor, action, detail->>'org', detail->>'cell' FROM loom_router.audit", + ) + == f"operator|place|{ORG}|first" + ) + assert await _sql(STANDBY, local_db, "SELECT org FROM loom.tenancy") == ORG From bb8e0412be154087030ecac7cc5c37f7bfbbca6a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 04:34:38 +0000 Subject: [PATCH 018/181] loom cell attach turns a prepared first cell into a serving one; prepare takes the same lock The attach command holds the attach lock on a direct, marked session of the cell's Postgres for its run and resumes by the directory row and the cell's mode. It refuses before any write on the stores' identity, the session, the roles, settings and subscriptions that would let an unchecked writer through, the database encoding, durability, its own grants, the ClickHouse writers, the table list and the processes' heartbeats; posts attach/begin; creates the triggers under the migration lock; in recording scans and writes the tenancy rows, refusing an org past ORG_MAX_BYTES; commits enforced, proves it durable, waits out every transaction that could have read recording, rescans, places every org in batches and marks the cell serving. A refusal after the mode commit leaves the cell enforced and attaching, and a rerun resumes there. Prepare refuses a session behind a pooler or a database that is not UTF8, and takes the attach lock. Doctor runs attach's Postgres reads again and reads stray orgs with --stray-orgs. instances.read_recent keeps stopped rows inside the widened window, and PRUNE_AFTER_S names the sweep's hour. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell.py | 4 +- server/cli/cell_attach.py | 756 +++++++++++++++++++ server/cli/cell_attach_beats.py | 324 ++++++++ server/cli/cell_attach_pg.py | 690 +++++++++++++++++ server/cli/cell_prepare.py | 42 +- server/cli/cell_scan.py | 50 +- server/cli/doctor.py | 47 +- server/cli/doctor_cell.py | 166 +++- server/core/cell_triggers.py | 139 ++-- server/core/instances.py | 57 +- tests/integration/test_cell_attach.py | 635 ++++++++++++++++ tests/integration/test_cell_scan.py | 2 +- tests/unit/test_cell_attach.py | 682 +++++++++++++++++ tests/unit/test_cell_org_triggers.py | 2 +- tests/unit/test_cell_prepare.py | 3 + tests/unit/test_doctor_client_checks.py | 2 +- tests/unit/test_doctor_client_client_only.py | 2 +- 17 files changed, 3529 insertions(+), 74 deletions(-) create mode 100644 server/cli/cell_attach.py create mode 100644 server/cli/cell_attach_beats.py create mode 100644 server/cli/cell_attach_pg.py create mode 100644 tests/integration/test_cell_attach.py create mode 100644 tests/unit/test_cell_attach.py diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..65f18990d8 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,7 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is -`server/cli/cell_prepare.py`. +`server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; `credential --cell --next | --finish | --revert` rotates a cell's. @@ -459,6 +459,8 @@ def _add_cell_parser(sub: Any) -> None: remove.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") remove.set_defaults(func=_cmd_remove) + from server.cli.cell_attach import add_attach_parser from server.cli.cell_prepare import add_prepare_parser add_prepare_parser(csub) + add_attach_parser(csub) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py new file mode 100644 index 0000000000..368f74ebf8 --- /dev/null +++ b/server/cli/cell_attach.py @@ -0,0 +1,756 @@ +"""`loom cell attach`: turn a prepared first cell into a `serving` one +(docs/architecture/cells.md, "Attaching a cell"; the runbook is +docs/operations/cells.md). + +A run holds the attach lock on a direct session of the cell's Postgres for its life, and +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: + +1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +2. the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, + the table list, durability, attach's own grants; +3. `attach/begin` (the router probes the stores and writes the fingerprints); +4. the mode: `recording`, or `enforced` on a resume; +5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; +7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode + only rises, so every later refusal leaves the cell `enforced` and `attaching`; +8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction + that could have read `recording`, the rescan, the tenancy rows and their placements + (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals +and the scan and prints what would be placed, writing nothing. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from collections.abc import Callable, Iterable +from typing import Any + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli import cell_scan +from server.cli._ch import writes +from server.cli.cell_ch import Refusal, Refused, read_topology +from server.core import cell, cell_triggers +from server.settings import settings + +_WAIT_S = 600 # not a setting: a flag's default +_SCAN_BUDGET_S = 7200 # not a setting: a flag's default +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on the lock session's catalog reads +_HEALTH_TIMEOUT = httpx.Timeout(10.0, connect=5.0) # not a setting: one health read + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return f"{args.cell!r} is no cell name (1 to 32 of a-z, 0-9, -)" + if args.cell != settings.cell: + return f"LOOM_CELL is {settings.cell!r}: attach runs with LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return ( + "a cell keeps its state in Postgres: set LOOM_STORAGE_AUTHORITY=postgres and " + "LOOM_PG_DSN" + ) + if args.wait < 0 or args.scan_budget <= 0: + return "--wait is 0 or more seconds, --scan-budget more than 0" + return "" + + +class Router: + """The router's operator API for one cell.""" + + def __init__( + self, url: httpx.URL, name: str, transport: httpx.AsyncBaseTransport | None + ) -> None: + self.url, self.name, self.transport = url, name, transport + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + timeout=cell_cli._OPERATOR_TIMEOUT, + trust_env=self.url.scheme == "https", + transport=self.transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" + url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) + try: + async with self._client() as client: + r = await (client.get(url) if body is None else client.post(url, json=body)) + except httpx.HTTPError as e: + raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None + answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused("router", cell_cli.not_the_router(r)) + if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") + if r.status_code != 200: + raise _refused( + "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" + ) + return answer + + async def health(self, host: str) -> str: + """The deployment id `https:///v1/health/deployment` answers ('' when none).""" + try: + async with httpx.AsyncClient( + timeout=_HEALTH_TIMEOUT, transport=self.transport + ) as client: + r = await client.get(f"https://{host}/v1/health/deployment") + answer = r.json() + except (httpx.HTTPError, ValueError): + return "" + return str(answer.get("deployment_id") or "") if isinstance(answer, dict) else "" + + +def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: + """Step 2's refusal of every org past `cell.ORG_MAX_BYTES`, before any tenancy row.""" + return [ + Refusal( + f"org of {len(org.encode())} bytes ({org[:64]}…)", + f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " + "remove it first (docs/operations/cells.md, the over-long org procedure)", + ) + for org, sources in sorted(found.too_long.items()) + ] + + +def _refused(subject: str, reason: str) -> Refused: + return Refused([Refusal(subject, reason)]) + + +def _encoded_size(body: Any) -> int: + return len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + + +def batches(orgs: Iterable[str], deployment_id: str) -> list[list[str]]: + """`orgs` in `attach/place` batches: at most ATTACH_BATCH each, and an encoded body + within the router's body bound.""" + from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES + + out: list[list[str]] = [] + batch: list[str] = [] + for org in sorted(orgs): + if batch and ( + len(batch) >= ATTACH_BATCH + or _encoded_size({"deployment_id": deployment_id, "orgs": [*batch, org]}) + > MAX_BODY_BYTES + ): + out.append(batch) + batch = [] + batch.append(org) + if batch: + out.append(batch) + return out + + +class Run: + """One attach run: its sessions, what it read, and what it prints.""" + + def __init__( + self, + ch: Any, + args: argparse.Namespace, + router: Router, + log: Callable[[str], None], + ) -> None: + self.ch, self.args, self.router, self.log = ch, args, router, log + self.name = args.cell + self.wait = apg.Wait(args.wait) + self.lock_conn: Any = None + self._lock_cm: Any = None + self.row: dict[str, Any] = {} + self.mode = "" + self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) + self.keys_fp = "" + self.report: dict[str, Any] = {"cell": self.name} + + # -- the lock session -------------------------------------------------------------- + + async def open_lock(self) -> None: + from server.core import pg + + self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self.lock_conn = await self._lock_cm.__aenter__() + if refusals := await apg.direct_refusals(self.lock_conn, "attach"): + raise Refused(refusals) + await apg.take_lock(self.lock_conn, "attach") + + async def close_lock(self) -> None: + if self._lock_cm is not None: + await self._lock_cm.__aexit__(None, None, None) + self._lock_cm = self.lock_conn = None + + async def confirm_lock(self) -> None: + """The lock is held still, or taken again (refusing when another run holds it now), + and the cell's state read again: a cleanup, a prepare rerun or another attach could + have run in the gap.""" + try: + if await apg.lock_held(self.lock_conn): + return + except Exception as e: # noqa: BLE001 — a lost session is taken again below + self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") + else: + self.log("the attach lock was lost; taking it again") + try: + await self.close_lock() + except Exception: # noqa: BLE001 — the old session is gone either way + self._lock_cm = self.lock_conn = None + await self.open_lock() + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): + raise _refused( + "cell state", + f"changed while the lock was lost (now {state}, first row {first}): rerun attach", + ) + row = await self.router.call("") + if ( + row.get("deployment_id") != self.row.get("deployment_id") + or row.get("state") != "attaching" + ): + raise _refused("directory row", "changed while the lock was lost: rerun attach") + + # -- the refusals ------------------------------------------------------------------ + + async def read_own_row(self) -> None: + own = await self.lock_conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("postgres loom.cell_self", "absent: run loom cell prepare first") + self.own_fp = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + + def judge_beats(self, stopped_unset: str) -> Callable[[list[beats.Beat]], beats.Verdict]: + def _judge(found: list[beats.Beat]) -> beats.Verdict: + if stopped_unset == "ignore": + found = [b for b in found if b.live or b.cell == self.name] + return beats.judge( + found, self.name, self.own_fp, "wait" if stopped_unset == "wait" else "refuse" + ) + + return _judge + + async def check_beats(self, stopped_unset: str) -> beats.Verdict: + verdict = await beats.settle( + lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait + ) + (self.keys_fp,) = verdict.keys_fps + return verdict + + async def stores_refusals(self) -> list[Refusal]: + from server.core import pg_profile + + out = [] + row = self.row + if not row.get("is_first") or row.get("state") != "attaching": + out.append( + Refusal( + "directory row", + f"is {'the first' if row.get('is_first') else 'a later'} cell, " + f"{row.get('state')}: this attach is the first cell's, while attaching", + ) + ) + if row.get("other"): + out.append( + Refusal( + "directory row", + f"another cell, {row['other']}, is registered: remove it first " + "(loom cell remove)", + ) + ) + expected = str(row.get("deployment_id") or "") + recorded_pg = await self.lock_conn.fetchval("SELECT deployment_id FROM loom.authority") + if recorded_pg != expected: + out.append( + Refusal( + "postgres", f"records deployment id {recorded_pg!r}, the directory {expected!r}" + ) + ) + recorded = await pg_profile.recorded_authority(self.ch) + if recorded is None or str(recorded["deployment_id"]) != expected: + out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) + state, first = await apg.read_mode(self.lock_conn) + if first != (True, self.name): + out.append( + Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") + ) + if state not in ("recording", "enforced"): + out.append(Refusal("postgres cell state", f"reads {state}")) + answered = await self.router.health(str(row.get("host") or "")) + if answered != expected: + out.append( + Refusal( + f"https://{row.get('host')}/v1/health/deployment", + f"answers deployment id {answered!r}, not {expected!r}", + ) + ) + return out + + async def durability_refusals(self) -> list[Refusal]: + from server.core import pg, pg_profile + + out = [] + verdict = pg_profile.durability_verdict("synchronous", *await pg_profile.standby_state()) + if verdict.refusal: + out.append(Refusal("postgres durability", verdict.refusal)) + try: + await pg.check_durable_message() + except Exception as e: # noqa: BLE001 — a role that may not write it refuses + out.append(Refusal("postgres durability", f"pg_logical_emit_message refused: {e}")) + if (now := await pg_profile.standby_now()) != "synchronous": + out.append( + Refusal("postgres standbys", f"commits are acknowledged as {now}, not synchronous") + ) + return out + + async def ch_limit_refusals(self) -> list[Refusal]: + """A profile or quota that caps the rows or bytes attach's user reads would end a + scan early.""" + row = await self.ch.query_one( + "SELECT getSetting('max_rows_to_read') AS r, getSetting('max_bytes_to_read') AS b" + ) + out = [] + if row and (int(row["r"]) or int(row["b"])): + out.append( + Refusal("clickhouse user", "a settings profile caps the rows or bytes it reads") + ) + quota = await self.ch.query_all( + "SELECT quota_name FROM system.quota_usage " + "WHERE max_read_rows > 0 OR max_read_bytes > 0" + ) + out += [ + Refusal(f"clickhouse quota {q['quota_name']}", "caps the rows or bytes attach reads") + for q in quota + ] + return out + + async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: + from server.cli import cell_writers + + topology = await read_topology(self.ch) + if refusals := topology.refusals(): + return refusals, topology, {} + names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + allowed = cell_writers.Allowed( + users=frozenset(self.args.ch_allow_user), + user_directories=frozenset(self.args.ch_allow_user_directory), + objects=frozenset(self.args.ch_allow_object), + replicas=frozenset(self.args.ch_allow_replica), + ) + return await cell_writers.check_writers(self.ch, topology, names, allowed), topology, names + + async def catalog_refusals(self) -> list[Refusal]: + """The Postgres checks that run again before the mode commit and in step 4.""" + holders = await apg.privilege_holders(self.lock_conn) + self.report["own_roles"] = holders.own + out = apg.privilege_refusals(holders, self.args.pg_allow_role) + out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) + return out + + async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: + """Every refusal before any write, named together.""" + out = await self.stores_refusals() + out += await apg.encoding_refusals(self.lock_conn) + out += await self.catalog_refusals() + out += await self.durability_refusals() + rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) + tables_pg = [f"loom.{r['relname']}" for r in rows] + out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await self.ch_limit_refusals() + if out: + raise Refused(out) + state, _ = await apg.read_mode(self.lock_conn) + # A stopped process with LOOM_CELL unset refuses while the cell can still be + # backed out; on a resume in `enforced` step 4 waits for it to age out. + await self.check_beats("refuse" if state == "recording" else "ignore") + writers, topology, names = await self.writer_refusals() + out += writers + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + out += listed.refusals + if out: + raise Refused(out) + return topology, names, listed + + # -- begin ------------------------------------------------------------------------- + + async def begin(self) -> None: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + from server.router.limits import ATTACH_BEGIN_MAX_BYTES + + current, nxt = self.own_fp + nonces = _new_nonces() + body = { + "deployment_id": self.row["deployment_id"], + "current_fp": current, + "next_fp": nxt, + "keys_fp": self.keys_fp, + "ch_allow_users": sorted(self.args.ch_allow_user), + "ch_allow_user_directories": sorted(self.args.ch_allow_user_directory), + "ch_allow_objects": sorted(self.args.ch_allow_object), + "ch_allow_replicas": sorted(self.args.ch_allow_replica), + "pg_allow_roles": sorted(self.args.pg_allow_role), + "nonces": nonces, + } + if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: + raise _refused( + "attach/begin", + f"its body is {size} bytes, past the router's {ATTACH_BEGIN_MAX_BYTES}: the " + "fingerprints or the allow lists are too large", + ) + try: + await _write_nonces(self.ch, nonces) + self.row = {**self.row, **await self.router.call("/attach/begin", body)} + finally: + await _delete_nonces(self.ch, nonces) + + # -- the scans --------------------------------------------------------------------- + + async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: + budget = cell_scan.Budget(self.args.scan_budget) + current, nxt = self.own_fp + return await cell_scan.scan( + self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) + ) + + def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: + return { + "malformed": {k: v[:20] for k, v in sorted(found.malformed.items())}, + "too_long": { + f"{o[:64]}… ({len(o.encode())} bytes)": sorted(s) for o, s in found.too_long.items() + }, + } + + async def triggers(self) -> None: + if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + raise Refused(refusals) + + # -- step 3 ------------------------------------------------------------------------ + + async def ready_for_commit(self) -> bool: + """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, + a booting process skips the try.""" + verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) + if verdict.refusals: + raise Refused(verdict.refusals) + return not verdict.booting + + async def commit_mode(self) -> None: + from server.core import pg + + await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + try: + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + ) + except Refused: + raise + except Exception as e: # noqa: BLE001 — the durable read says whether it landed + self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.end_mode_transactions(conn, self.confirm_lock) + + async def durable(self) -> int: + """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + xmax, above the read's own xid.""" + from server.core import pg_profile + + while True: + try: + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read() + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") + break + except Refused: + raise + except Exception as e: + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused("postgres durable read", "did not commit within --wait") from e + if read.first != (True, self.name) or read.mode != "enforced": + raise _refused( + "postgres cell state", + f"the durable read saw mode {read.mode!r} and first row {read.first}: the mode " + "commit did not land; rerun attach", + ) + self.mode = "enforced" + xmax = await apg.snapshot_xmax(self.lock_conn) + if xmax <= read.xid: + raise _refused("postgres snapshot", f"its xmax {xmax} is not above {read.xid}") + return xmax + + # -- step 4 ------------------------------------------------------------------------ + + async def cheap_checks(self, stopped_unset: str) -> None: + from server.core import pg_profile + + out, _, _ = await self.writer_refusals() + out += [ + Refusal("postgres triggers", d.text) + for d in await cell_triggers.missing(self.lock_conn) + ] + out += await self.catalog_refusals() + if out: + raise Refused(out) + await self.check_beats(stopped_unset) + if (now := await pg_profile.standby_now()) != "synchronous": + raise _refused("postgres standbys", f"commits are acknowledged as {now}") + + async def final_checks(self, watch: beats.Watch) -> None: + from server.core import instances + + floor = apg.Wait(max(self.wait.left(), instances.boot_allowance_s())) + + async def _read() -> list[beats.Beat]: + found = await watch.read_once() + if refusals := watch.refusals(): + raise Refused(refusals) + # A cell-correct process whose cell row alone was swept. + return [b for b in found if not (b.cell is None and b.instance_id in watch.cell_ids)] + + verdict = await beats.settle( + _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() + ) + if verdict.keys_fps != {self.keys_fp}: + raise _refused( + "processes", + "the root-secret fingerprint changed during attach (a root-secret rotation): " + "rerun attach", + ) + + async def step4(self, xmax: int, topology: Any, names: Any) -> None: + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await apg.wait_transactions(conn, xmax, self.wait) + await self.cheap_checks("wait") + since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) + watch = beats.Watch(self.ch, since, self.name) + if refusals := watch.refusals_up_front(): + raise Refused(refusals) + task = asyncio.create_task(watch.run()) + try: + listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) + found = await self.scan(topology, listed) + orgs = set(found.orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + placed = 0 + for batch in batches(orgs, self.row["deployment_id"]): + await self.confirm_lock() + answer = await self.router.call( + "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} + ) + placed += int(answer.get("placed") or 0) + self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) + writers, _, _ = await self.writer_refusals() + if writers: + raise Refused(writers) + finally: + task.cancel() + await self.final_checks(watch) + row = await self.router.call("") + current, nxt = self.own_fp + if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + raise _refused( + "postgres loom.cell_self", + "differs from the directory's fingerprints (a loom cell update between them?)", + ) + state, _ = await apg.read_mode(self.lock_conn) + if state != "enforced": + raise _refused("postgres cell state", f"reads {state}, not enforced") + + async def serving(self) -> dict[str, Any]: + from server.cli.cell_prepare import _delete_nonces, _new_nonces, _write_nonces + + nonces = _new_nonces() + try: + await _write_nonces(self.ch, nonces) + return await self.router.call( + "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} + ) + finally: + await _delete_nonces(self.ch, nonces) + + # -- the run ----------------------------------------------------------------------- + + async def done(self) -> bool: + """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + state, _ = await apg.read_mode(conn) + if state != "enforced": + raise _refused( + "postgres cell state", + f"the directory says serving and the cell reads {state}: run loom cell restore", + ) + return True + + async def run(self) -> int: + self.row = await self.router.call("") + if self.row.get("state") == "serving": + await self.done() + self.report.update(state="serving", done=True) + return 0 + await self.open_lock() + try: + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") + await self.triggers() + await self.precommit(found) + await self.commit_mode() + else: + await self.triggers() + await self.router.call( + "/attach/enforced", {"deployment_id": self.row["deployment_id"]} + ) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 + finally: + await self.close_lock() + + async def precommit(self, found: cell_scan.ScanResult) -> None: + from server.core import pg_profile + + out = await self.catalog_refusals() + if (now := await pg_profile.standby_now()) != "synchronous": + out.append(Refusal("postgres standbys", f"commits are acknowledged as {now}")) + if found.elapsed_s > self.args.scan_budget / 2: + out.append( + Refusal( + "--scan-budget", + f"step 2 took {found.elapsed_s:.0f} s, more than half of " + f"{self.args.scan_budget} s, and step 4 reads the same tables again: " + "raise it", + ) + ) + if out: + raise Refused(out) + verdict = await self.check_beats("refuse") + if refusals := beats.final_refusals(verdict): + raise Refused(refusals) + + async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: + found = await self.scan(topology, listed) + per_source: dict[str, list[str]] = {} + for org, sources in found.orgs.items(): + for s in sources: + per_source.setdefault(s, []).append(org) + self.report.update( + dry_run=True, + scan_seconds=round(found.elapsed_s, 1), + half_budget_ok=found.elapsed_s <= self.args.scan_budget / 2, + orgs=len(found.orgs), + per_source={ + s: {"count": len(o), "sample": sorted(o)[:5]} for s, o in sorted(per_source.items()) + }, + skipped=self._named(found), + ) + return 0 + + +async def attach( + ch: Any, + args: argparse.Namespace, + url: httpx.URL, + transport: httpx.AsyncBaseTransport | None = None, +) -> int: + def log(text: str) -> None: + print(f"loom cell attach: {text}", file=sys.stderr) + + run = Run(ch, args, Router(url, args.cell, transport), log) + started = time.monotonic() + try: + code = await run.run() + except Refused as e: + for r in e.refusals: + log(f"refusing: {r}") + mode = ( + "enforced, attaching: rerun attach to resume" + if run.mode == "enforced" + else ("unchanged" if not run.mode else f"{run.mode}") + ) + log(f"the cell stays {mode}") + return 1 + run.report["seconds"] = round(time.monotonic() - started, 1) + print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) + return code + + +def cmd_attach(args: argparse.Namespace) -> int: + if problem := _args_problem(args): + print(f"loom cell attach: {problem}", file=sys.stderr) + return 2 + url = cell_cli.operator_router(args) + if isinstance(url, str): + print(f"loom cell attach: {url}", file=sys.stderr) + return 2 + from server.cli._ch import run_connected + from server.core.ch import statement_timeout + + async def _run(ch: Any) -> int: + try: + return await attach(ch, args, url) + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + print( + f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", + file=sys.stderr, + ) + return 1 + + # Every ClickHouse statement, the connect included, may run as long as a scan may. + with statement_timeout(args.scan_budget): + return run_connected(_run, org_free=writes("attaches the cell; places the orgs it holds")) + + +def add_attach_parser(csub: Any) -> None: + p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") + p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Run the refusals and the scan and print what would be placed; write nothing " + "(the router's own begin refusals do not run)", + ) + for flag, what in ( + ("--ch-allow-user", "A ClickHouse user that may write the configured databases"), + ("--ch-allow-user-directory", "A ClickHouse user directory the check cannot read"), + ("--ch-allow-object", "A ClickHouse object (db.name) that writes the databases"), + ("--ch-allow-replica", "A replica outside the cluster (db.table:replica)"), + ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), + ): + p.add_argument(flag, action="append", default=[], help=what) + p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py new file mode 100644 index 0000000000..a2fc37701d --- /dev/null +++ b/server/cli/cell_attach_beats.py @@ -0,0 +1,324 @@ +"""The first attach's judgment of the cell's processes, from their heartbeats +(docs/architecture/cells.md, "Attaching a cell" and "The heartbeat's keys"). + +Every check reads both heartbeat tables across every deployment sharing the ClickHouse, as +prepare does, stopped rows included (`instances.read_recent`), and judges in one order: + +- `cell` first: a live process whose `LOOM_CELL` is unset or another cell refuses at once; + a stopped one, or a row with no cell row that is no longer live, may still be writing + for the widened window after its stop beat, and is refused, or waited on until it ages + out, as the caller's `stopped_unset` says; +- then the keys a current build reports (`serves`, `marked`, `router`): a cell row + without them refuses, even while booting; +- then booting: a live instance with no cell row yet, no tenancy mode yet, `marked` null, + or `router` null where `serves` is true, is waited on; `marked` false, or `router` false + where `serves` is true, refuses, naming the process and the router's cause; +- and the fingerprint: each live process's matches `loom.cell_self`'s current or next. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from collections.abc import Awaitable, Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from server.cli.cell_attach_pg import Wait +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, instances + +StoppedUnset = Literal["refuse", "wait"] + + +@dataclass(frozen=True) +class Beat: + """One process: its instance row (`live`: inside the live window and not stopped) and + its cell row (`cell` None when it wrote none), the fingerprint JSON decoded.""" + + instance_id: str + live: bool + stopped: bool + keys_fp: str + last_beat: str + cell: str | None = None + mode: str = "" + value: dict[str, Any] | None = None + + +async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: + rows = await instances.read_recent(ch, since=since) + ids = [str(r["instance_id"]) for r in rows] + cells = await ch.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": ids}, + ) + by_id = {str(r["instance_id"]): r for r in cells} + out = [] + for r in rows: + found = by_id.get(str(r["instance_id"])) + beat = Beat( + instance_id=str(r["instance_id"]), + live=bool(r["live"]), + stopped=bool(r["stopped"]), + keys_fp=str(r["keys_fp"]), + last_beat=str(r["last_beat"]), + cell=None if found is None else str(found["cell"]), + mode="" if found is None else str(found["tenancy_mode"]), + value=None if found is None else _decoded(str(found["fingerprint"])), + ) + out.append(beat) + return out + + +def _decoded(text: str) -> dict[str, Any] | None: + try: + value = json.loads(text) + except ValueError: + return None + return value if isinstance(value, dict) else None + + +@dataclass +class Verdict: + """What one read found: the refusals, the processes waited on (booting, or a stopped + process with `LOOM_CELL` unset aging out), the live processes judged, their `keys_fp` + values, and whether one reports SCIM on.""" + + refusals: list[Refusal] = field(default_factory=list) + booting: list[str] = field(default_factory=list) + aging: list[str] = field(default_factory=list) + live: int = 0 + keys_fps: set[str] = field(default_factory=set) + scim: bool = False + + +def judge( + beats: Iterable[Beat], + name: str, + accepted: Iterable[Mapping[str, Any] | None], + stopped_unset: StoppedUnset, +) -> Verdict: + """The verdict on `beats` for the cell `name`, whose processes run `accepted` (its + `loom.cell_self` current and next).""" + values = [v for v in accepted if v is not None] + out = Verdict() + for b in beats: + who = f"process {b.instance_id}" + if b.cell is None: + if b.live: + out.booting.append(b.instance_id) + else: + _stopped(out, b, "stopped before reporting its LOOM_CELL", stopped_unset) + continue + if b.cell != name: + runs = f"runs with LOOM_CELL={b.cell}" if b.cell else "runs with LOOM_CELL unset" + if b.live: + out.refusals.append(Refusal(who, f"{runs}: set LOOM_CELL={name} on it")) + else: + _stopped(out, b, f"{runs} and stopped", stopped_unset) + continue + if not b.live: + continue + value = b.value or {} + if not all(k in value for k in ("serves", "marked", "router")): + out.refusals.append( + Refusal(who, "reports no serves, marked or router key: upgrade it to this release") + ) + continue + if value["marked"] is False: + out.refusals.append( + Refusal( + who, + "its Postgres sessions lose the tenancy marker (a pooler?): point " + "LOOM_PG_DSN straight at Postgres", + ) + ) + continue + serves = value["serves"] is True + if serves and value["router"] is False: + why = value.get("router_why") or "no cause reported" + out.refusals.append(Refusal(who, f"cannot read its router: {why}")) + continue + if value["marked"] is None or (serves and value["router"] is None) or not b.mode: + out.booting.append(b.instance_id) + continue + fields = value.get("fields") + if not isinstance(fields, dict) or not any(cell.matches(fields, v) for v in values): + out.refusals.append( + Refusal(who, "runs neither loom.cell_self's current nor its next configuration") + ) + continue + out.live += 1 + out.keys_fps.add(b.keys_fp) + out.scim = out.scim or bool(fields.get("scim")) + return out + + +def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> None: + if stopped_unset == "wait": + out.aging.append(b.instance_id) + return + out.refusals.append( + Refusal( + f"process {b.instance_id}", + f"{what} at {b.last_beat}, and may still be writing (its drain and retirement " + "after its stop beat)", + ) + ) + + +def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: + out = [] + if not verdict.live and not verdict.booting: + out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) + if len(verdict.keys_fps) > 1: + out.append( + Refusal( + "processes", + "the live processes report different root-secret fingerprints (keys_fp): " + "finish the root-secret rollout first", + ) + ) + if verdict.scim and not scim_ok: + out.append( + Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") + ) + return out + + +async def settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + *, + extend_s: float = 0.0, +) -> Verdict: + """Read and judge until nothing is waited on: Refused at the first refusal, and once + `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended + once, to `extend_s` past the first sight of the last process first seen booting.""" + first_seen: dict[str, float] = {} + extended = False + while True: + verdict = judge_beats(await read()) + if verdict.refusals: + raise Refused(verdict.refusals) + if not verdict.booting and not verdict.aging: + if refusals := final_refusals(verdict): + raise Refused(refusals) + return verdict + for i in verdict.booting: + first_seen.setdefault(i, wait.spent) + if await wait.pause(): + continue + if extend_s and not extended and verdict.booting: + extended = True + wait.seconds = max(wait.seconds, max(first_seen[i] for i in verdict.booting) + extend_s) + if await wait.pause(): + continue + raise Refused( + [Refusal(f"process {i}", "still booting at the end of --wait") for i in verdict.booting] + + [ + Refusal(f"process {i}", "stopped with LOOM_CELL unset, not aged out within --wait") + for i in verdict.aging + ] + ) + + +# -- the rescan's watch ---------------------------------------------------------------- + +_READ_TIMEOUT_S = 30.0 # not a setting: one heartbeat read's bound +_SKEW_S = 10.0 # not a setting: a margin for clock skew between ClickHouse hosts + + +def watch_period_s() -> float: + """`T'`: how long a row whose window reaches past the rescan's start is kept before the + leader's sweep may drop it, less a margin for clock skew.""" + return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + + +class Watch: + """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's + start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at + the end even once the leader swept its rows.""" + + def __init__( + self, ch: Any, since: str, name: str, clock: Callable[[], float] = time.monotonic + ) -> None: + self.ch, self.since, self.name = ch, since, name + self._clock = clock + self.cell_ids: set[str] = set() + self.unset: dict[str, str] = {} + self.no_cell: dict[str, str] = {} + self.gap = False + self._good = clock() + + def refusals_up_front(self) -> list[Refusal]: + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + async def read_once(self) -> list[Beat]: + try: + async with asyncio.timeout(_READ_TIMEOUT_S): + beats = await read_beats(self.ch, since=self.since) + except Exception: + self._check_gap() + raise + self._check_gap() + self._good = self._clock() + for b in beats: + if b.cell == self.name: + self.cell_ids.add(b.instance_id) + elif b.cell is not None: + self.unset[b.instance_id] = b.last_beat + elif b.stopped: + self.no_cell[b.instance_id] = b.last_beat + return beats + + def _check_gap(self) -> None: + if self._clock() - self._good > watch_period_s(): + self.gap = True + + async def run(self) -> None: + """Read until cancelled; a failed read is retried at the next period.""" + while True: + await asyncio.sleep(watch_period_s() / 2) + try: + await self.read_once() + except Exception: # noqa: BLE001 — a failed read is retried; a gap refuses + continue + + def refusals(self) -> list[Refusal]: + out = [ + Refusal( + f"process {i}", + f"ran with LOOM_CELL unset or another cell during the rescan (last beat {at})", + ) + for i, at in sorted(self.unset.items()) + ] + out += [ + Refusal( + f"process {i}", + f"stopped during the rescan before reporting its LOOM_CELL (last beat {at})", + ) + for i, at in sorted(self.no_cell.items()) + if i not in self.cell_ids + ] + if self.gap: + out.append( + Refusal( + "heartbeats", + "not read for longer than the leader keeps a stopped process's rows: " + "rerun attach", + ) + ) + return out diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py new file mode 100644 index 0000000000..6ee4432907 --- /dev/null +++ b/server/cli/cell_attach_pg.py @@ -0,0 +1,690 @@ +"""The first attach's reads and writes on the cell's Postgres (docs/architecture/cells.md, +"Attaching a cell"): the session checks, the one-run lock, the catalog checks attach and +`loom doctor` share (the roles that can make an org held or lower the mode, the settings +and subscriptions that pass an unchecked writer), the trigger pass, the mode commit, its +durable read and the wait for the transactions that could have read `recording`. + +Every check returns its refusals as `cell_ch.Refusal` (subject, reason), as the org scan +and the ClickHouse writer check do, so attach and doctor print the same names.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.cli.cell_ch import Refusal, Refused +from server.core import cell, cell_triggers, pg +from server.core.cell import ATTACH_LOCK_KEY + +_sleep = asyncio.sleep # the seam the wait tests replace + +# The pause between polls of a wait. +POLL_S = 1.0 # not a setting: a poll interval +# The migration lock's wait a trigger try (a boot holds it for its migrations). +_MIGRATION_LOCK_TIMEOUT = "10s" +# The mode commit's lock wait a try, and the statement bound above it. +_MODE_LOCK_TIMEOUT = "2s" +_MODE_STATEMENT_TIMEOUT = "5s" +# The server ends the lock session soon after its client is gone, releasing the lock. +_KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} + + +class Wait: + """`--wait`: the seconds attach's waits share, counting only the time spent waiting + (never scan time).""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self.spent = 0.0 + self._clock = clock + + def left(self) -> float: + return max(0.0, self.seconds - self.spent) + + def now(self) -> float: + return self._clock() + + async def pause(self, seconds: float = POLL_S) -> bool: + """Wait `seconds` (at most what is left) and count them; False once none is left.""" + step = min(seconds, self.left()) + if step <= 0: + return False + start = self._clock() + await _sleep(step) + self.spent += max(step, self._clock() - start) + return True + + +def _split(key: int) -> str: + """The `pg_locks` columns that hold a bigint advisory key, in this database.""" + return ( + f"l.locktype = 'advisory' AND l.classid = ({key} >> 32)::oid " + f"AND l.objid = ({key} & 4294967295)::oid AND l.objsubid = 1 " + "AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())" + ) + + +_LOCK_HOLDER = f""" +SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted +""" + +_IDENTITY = ( + "SELECT pg_backend_pid() AS pid, backend_start FROM pg_stat_activity " + "WHERE pid = pg_backend_pid()" +) + + +async def direct_refusals(conn: Any, what: str) -> list[Refusal]: + """Whether `conn` is one direct, marked session: the same backend across two + transactions, and the tenancy marker kept through `RESET ALL` (a pooler that strips + startup parameters drops it). Behind a transaction pooler a session-level lock would + outlive the CLI on the pooler's server connection, and attach's tenancy rows would be + refused as an unchecked writer's.""" + async with conn.transaction(): + first = await conn.fetchrow(_IDENTITY) + async with conn.transaction(): + second = await conn.fetchrow(_IDENTITY) + out = [] + if first is None or second is None or tuple(first) != tuple(second): + out.append( + Refusal( + "postgres session", + f"{what}'s session moved between backends: point LOOM_PG_DSN straight at " + "Postgres, never through a pooler", + ) + ) + await conn.execute("RESET ALL") # a startup value survives it, a SET does not + if await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) != "on": + out.append( + Refusal( + "postgres session", + f"{what}'s session loses {pg.TENANCY_MARKER}: run it with LOOM_CELL set, " + "LOOM_PG_DSN straight at Postgres", + ) + ) + return out + + +_HELD = f""" +SELECT EXISTS (SELECT 1 FROM pg_locks l + WHERE {_split(ATTACH_LOCK_KEY)} AND l.granted AND l.pid = pg_backend_pid()) +""" + + +async def lock_holder(conn: Any) -> str: + """Who holds the attach lock ('' when no one), never a query's text.""" + rows = await conn.fetch(_LOCK_HOLDER) + return "; ".join( + f"pid {r['pid']} ({r['application_name'] or 'no name'}, from {r['client'] or 'local'}, " + f"since {r['backend_start']})" + for r in rows + ) + + +async def take_lock(conn: Any, what: str) -> None: + """Take the attach lock on `conn` for the session's life, confirmed held by this + backend; Refused naming its holder when another run holds it.""" + for name, value in _KEEPALIVE.items(): + await conn.execute(f"SET {name} = {value}") + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", ATTACH_LOCK_KEY): + holder = await lock_holder(conn) or "a session that just ended" + raise Refused( + [ + Refusal( + "attach lock", + f"held by {holder}: an attach, prepare or update is running; {what} " + "runs one at a time", + ) + ] + ) + if not await lock_held(conn): + raise Refused( + [ + Refusal( + "attach lock", + "taken but not held by this session's backend: a pooler between " + "this command and Postgres; point LOOM_PG_DSN straight at it", + ) + ] + ) + + +async def lock_held(conn: Any) -> bool: + """Whether `conn`'s backend holds the attach lock.""" + return bool(await conn.fetchval(_HELD)) + + +async def encoding_refusals(conn: Any) -> list[Refusal]: + """A cell's database is UTF8: its `text` then holds no NUL or invalid UTF-8, so every + tenancy row within the length limit is an org a request can name.""" + encoding = await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()" + ) + if encoding == "UTF8": + return [] + return [ + Refusal( + "postgres database", + f"its encoding is {encoding}, not UTF8: a cell's database is UTF8 " + "(recreate it with ENCODING 'UTF8' and restore into it)", + ) + ] + + +# -- who can make an org held, or lower the mode -------------------------------------- + +_SYSTEM_SCHEMA = ( + "n.nspname NOT IN ('pg_catalog', 'information_schema') " + "AND n.nspname NOT LIKE 'pg\\_toast%' AND n.nspname NOT LIKE 'pg\\_temp\\_%'" +) + +# Every right a login role may hold that lets it make an org held, lower the mode, or run +# its own SQL as the servers on their marked sessions: each row a role and why. +_RIGHTS = f""" +WITH owned AS ( + SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + UNION ALL + SELECT p.proowner, 'owns loom.' || p.proname || '()' FROM pg_proc p + WHERE p.oid IN (to_regprocedure('loom.tenancy_refuse_unheld()'), + to_regprocedure('loom.tenancy_refuse_unmarked()'), + to_regprocedure('loom.cell_first_immutable()')) + UNION ALL + SELECT d.datdba, 'owns the database' FROM pg_database d + WHERE d.datname = current_database() + UNION ALL + SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} +), +login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) +SELECT r.rolname AS role, w.why FROM login r, LATERAL ( + SELECT 'may insert into loom.tenancy' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') + UNION ALL SELECT 'may update loom.tenancy (org)' + WHERE has_column_privilege(r.oid, 'loom.tenancy', 'org', 'UPDATE') + UNION ALL SELECT 'may update loom.tenancy_mode (mode)' + WHERE has_column_privilege(r.oid, 'loom.tenancy_mode', 'mode', 'UPDATE') + UNION ALL SELECT 'may insert into loom.tenancy_mode' + WHERE has_any_column_privilege(r.oid, 'loom.tenancy_mode', 'INSERT') + UNION ALL SELECT 'holds TRIGGER on loom.' || c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND has_table_privilege(r.oid, c.oid, 'TRIGGER') + UNION ALL SELECT 'may create in the database' + WHERE has_database_privilege(r.oid, current_database(), 'CREATE') + UNION ALL SELECT 'may create in schema ' || n.nspname FROM pg_namespace n + WHERE {_SYSTEM_SCHEMA} AND has_schema_privilege(r.oid, n.oid, 'CREATE') + UNION ALL SELECT 'may set session_replication_role' + WHERE has_parameter_privilege(r.oid, 'session_replication_role', 'SET') + OR has_parameter_privilege(r.oid, 'session_replication_role', 'ALTER SYSTEM') + UNION ALL SELECT 'may ALTER SYSTEM SET {pg.TENANCY_MARKER}' + WHERE has_parameter_privilege(r.oid, '{pg.TENANCY_MARKER}', 'ALTER SYSTEM') + UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g + WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') + AND pg_has_role(r.oid, g.oid, 'MEMBER') + UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' + END + FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') +) w(why) +ORDER BY 1, 2 +""" + +# The roles attach counts as its own: superusers, and a login role that itself owns +# loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the +# triggers, or lower the mode, whatever else it holds). +_OWN_ROLES = """ +SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( + SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') + AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) +)) OR rolname = current_user +ORDER BY 1 +""" + + +@dataclass(frozen=True) +class Holders: + """The login roles holding a right `_RIGHTS` counts: the ones attach counts as its own + (`own`, named in its output) and every other with its reasons.""" + + own: list[str] + others: dict[str, list[str]] + + +async def privilege_holders(conn: Any) -> Holders: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + others: dict[str, list[str]] = {} + for r in await conn.fetch(_RIGHTS): + if r["role"] not in own: + others.setdefault(str(r["role"]), []).append(str(r["why"])) + return Holders(own, others) + + +def privilege_refusals(holders: Holders, allowed: Iterable[str]) -> list[Refusal]: + allow = set(allowed) + return [ + Refusal( + f"postgres role {role}", + f"{'; '.join(why)}: it can make an org held or lower the mode outside attach; " + "revoke it, or name the role with --pg-allow-role", + ) + for role, why in sorted(holders.others.items()) + if role not in allow + ] + + +# -- what passes an unchecked writer through the triggers ----------------------------- + +_ROLE_SETTINGS = f""" +SELECT coalesce(r.rolname, 'every role') AS role, coalesce(d.datname, 'every database') AS db, + cfg +FROM pg_db_role_setting s +LEFT JOIN pg_roles r ON r.oid = s.setrole +LEFT JOIN pg_database d ON d.oid = s.setdatabase, +LATERAL unnest(s.setconfig) cfg +WHERE s.setdatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database())) + AND lower(split_part(cfg, '=', 1)) IN ('{pg.TENANCY_MARKER}', 'session_replication_role') +ORDER BY 1, 2, 3 +""" + +_SUBSCRIPTIONS = """ +SELECT subname, subenabled FROM pg_subscription +WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = current_database()) +ORDER BY 1 +""" + + +async def unmarked_refusals(conn: Any, session_passes: bool) -> list[Refusal]: + """What lets a session that sends no marker pass the triggers: the session read + (`session_passes`, doctor's unmarked session), an `ALTER ROLE … SET` or + `ALTER DATABASE … SET` of the marker or `session_replication_role` for this database or + every one, and any logical-replication subscription into this database (its apply + worker runs as `replica`, firing no origin trigger).""" + out = [] + if session_passes: + out.append( + Refusal( + "postgres unmarked session", + "a session that sends no marker is marked, or runs as replica (a role, " + "database, ALTER SYSTEM or options setting): every unchecked writer passes", + ) + ) + for r in await conn.fetch(_ROLE_SETTINGS): + out.append( + Refusal( + f"postgres setting for {r['role']} in {r['db']}", + f"{r['cfg']}: an unchecked writer under it passes the triggers; reset it", + ) + ) + for r in await conn.fetch(_SUBSCRIPTIONS): + state = "enabled" if r["subenabled"] else "disabled" + out.append( + Refusal( + f"postgres subscription {r['subname']}", + f"{state}: its apply worker fires no origin trigger; drop it", + ) + ) + return out + + +async def session_passes() -> bool: + """Whether a session that sends no marker passes the triggers all the same: marked, + or in the replica role that fires no origin trigger.""" + async with pg.dedicated(statement_timeout_ms=5000, marked=False) as conn: + value = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) + role = await conn.fetchval("SELECT current_setting('session_replication_role')") + return bool(value == "on" or role == "replica") + + +# -- attach's own role ------------------------------------------------------------------ + +_OWN_GRANTS = """ +SELECT need, held FROM (VALUES + ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), + ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), + ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), + ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), + ('UPDATE on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'UPDATE')), + ('SELECT on loom.cell_self', has_table_privilege('loom.cell_self', 'SELECT')), + ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), + ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), + ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', + has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) +) v(need, held) +WHERE NOT held +""" + + +async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: + """What attach's own role lacks (the guarded tables' owner roles are `missing`'s + `owner` defects), and a server that allows prepared transactions.""" + role = await conn.fetchval("SELECT current_user") + lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] + for table in tables: + if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): + lacks.append(f"SELECT on {table}") + out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] + out += await prepared_refusals(conn) + return out + + +async def prepared_refusals(conn: Any) -> list[Refusal]: + setting = await conn.fetchval("SELECT current_setting('max_prepared_transactions')::int") + if setting: + return [ + Refusal( + "postgres max_prepared_transactions", + f"is {setting}: a prepared transaction outlives every session, so step 4 " + "could not wait for it; set it to 0", + ) + ] + return [] + + +# -- the triggers ----------------------------------------------------------------------- + +_MIGRATION_LOCK = "SELECT pg_advisory_lock($1)" +_MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" + + +async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: + """Give every planned table its triggers (`cell_triggers.ensure_table` with + `recording_first`), one try at a time under the migration lock, retrying a table whose + lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield + by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + deadlock_ms = int( + await conn.fetchval( + "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " + "FROM pg_settings WHERE name = 'deadlock_timeout'" + ) + ) + out: list[Refusal] = [] + for table in await cell_triggers.planned_tables(conn): + yielded = False + while True: + lock_timeout = cell_triggers.LOCK_TIMEOUT + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_timeout, yielded = f"{deadlock_ms + 500}ms", True + done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) + if done: + break + if not await wait.pause(): + out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) + break + log(f"loom.{table}: its lock is busy; retrying") + if out: + return out + return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] + + +async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: + await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") + try: + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) == "55P03": + return False # a boot migrating: the next try + raise + finally: + await conn.execute("RESET lock_timeout") + try: + return await cell_triggers.ensure_table( + conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + ) + finally: + await conn.execute(_MIGRATION_UNLOCK, key) + + +def _busy_refusal(table: str, holders: list[Any]) -> Refusal: + named = "; ".join( + f"pid {h['pid']} ({'holds' if h['granted'] else 'waiting'}, {h['backend_type']}, " + f"{h['application_name'] or 'no name'}, transaction since {h['xact_start']}" + + (", an anti-wraparound vacuum, which never yields" if h["wraparound"] else "") + + ")" + for h in holders + ) + return Refusal( + f"postgres loom.{table}", + f"its triggers' lock was not taken within --wait; held by {named or 'no one now'}", + ) + + +# -- step 3: the mode commit ------------------------------------------------------------ + +# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on +# loom.tenancy_mode in this database, which only attach takes. +_MODE_HOLDERS = """ +SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted +FROM pg_locks l +JOIN pg_stat_activity a ON a.pid = l.pid +LEFT JOIN pg_roles r ON r.oid = a.usesysid +WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass + AND l.mode = 'ExclusiveLock' + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.pid <> pg_backend_pid() +""" + +_TIMED_OUT = frozenset({"55P03", "57014"}) + + +async def commit_enforced( + conn: Any, + confirm_lock: Callable[[], Any], + ready: Callable[[], Any], + wait: Wait, + log: Callable[[str], None], +) -> None: + """Raise the mode to `enforced` on `conn` (a session of its own, never the lock + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, + within `wait`; `confirm_lock` before each try and each termination (it refuses when + another run holds the attach lock), and `ready` before each try (False skips the try + while a process boots). After a failed try the in-flight mode transaction, found by its + lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the + durable read's to say.""" + while True: + await confirm_lock() + if await ready(): + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", f"the mode update read {status}" + ) + ] + ) + return + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + log("the mode's lock is busy; retrying") + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads it " + "FOR SHARE; rerun attach", + ) + ] + ) + + +async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: + """End an earlier run's mode transaction still in flight and wait until its lock + clears; Refused when its role is a superuser and attach's is not.""" + for _ in range(30): + rows = await conn.fetch(_MODE_HOLDERS) + if not rows: + return + superuser = bool( + await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") + ) + for r in rows: + if r["rolsuper"] and not superuser: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"an earlier mode transaction ({r['application_name'] or 'no name'}, " + f"since {r['xact_start']}) runs as a superuser: a superuser must " + "end it (pg_terminate_backend)", + ) + ] + ) + await confirm_lock() + await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) + await _sleep(0.1) + + +# -- the durable read ------------------------------------------------------------------- + +_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried + + +@dataclass(frozen=True) +class Durable: + """What the durable read saw: the mode, the `first` row, and the xid its logical + message took.""" + + mode: str + first: tuple[bool, str] | None + xid: int + + +async def durable_read() -> Durable: + """The mode and the `first` row read in a transaction that emits a transactional + logical message, whose commit waits for the synchronous standbys: once it returns, the + mode read is on them. Refused when the message took no xid.""" + async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") + xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + +async def snapshot_xmax(conn: Any) -> int: + return int(await conn.fetchval("SELECT pg_snapshot_xmax(pg_current_snapshot())::text")) + + +# -- step 4: the transactions that could have read `recording` -------------------------- + +# Every backend but this statement's own: of the cell's database, one whose xid or (but +# an autovacuum's) snapshot is older than X; of another database, one whose xid is older +# (it holds every snapshot's horizon). Both compared through age() against one reference, +# which is exactly `< X`, wraparound-safe, and takes no xid. +_OPEN = """ +WITH x AS (SELECT age($1::text::xid8::xid) AS a) +SELECT s.pid, s.backend_type, s.application_name, s.xact_start::text AS xact_start, + s.datid = (SELECT oid FROM pg_database WHERE datname = current_database()) AS ours, + coalesce(s.backend_xid IS NOT NULL AND age(s.backend_xid) > x.a, false) AS xid_old, + coalesce(s.backend_xmin IS NOT NULL AND age(s.backend_xmin) > x.a + AND s.backend_type <> 'autovacuum worker', false) AS xmin_old, + coalesce(s.query = '', false) AS hidden +FROM pg_stat_activity s, x +WHERE s.pid <> pg_backend_pid() +""" + + +def _named(r: Any) -> str: + return ( + f"pid {r['pid']} ({r['backend_type']}, {r['application_name'] or 'no name'}, " + f"transaction since {r['xact_start']})" + ) + + +async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: + """Wait until no backend of the cell's database holds an xid or a snapshot older than + `xmax`; Refused at once on a prepared transaction or an unreadable row, and past + `wait` naming each backend waited on and each other database's horizon holder.""" + while True: + if refusals := await prepared_refusals(conn): + raise Refused(refusals) + if await conn.fetchval("SELECT count(*) FROM pg_prepared_xacts"): + raise Refused( + [Refusal("postgres prepared transaction", "one is open: commit or roll it back")] + ) + rows = await conn.fetch(_OPEN, str(xmax)) + hidden = [r for r in rows if r["ours"] and r["hidden"]] + if hidden: + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + "its transaction state is hidden from attach's role: grant it " + "pg_read_all_stats", + ) + for r in hidden + ] + ) + waited = [r for r in rows if r["ours"] and (r["xid_old"] or r["xmin_old"])] + if not waited: + return + if not await wait.pause(): + horizon = [r for r in rows if not r["ours"] and r["xid_old"]] + raise Refused( + [ + Refusal(f"postgres {_named(r)}", "may have read `recording`: still open") + for r in waited + ] + + [ + Refusal( + f"postgres {_named(r)}", + "another database's transaction, holding the snapshot horizon", + ) + for r in horizon + ] + ) + + +# -- the tenancy rows ------------------------------------------------------------------- + + +async def write_tenancy(orgs: Iterable[str]) -> int: + """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the + rows written.""" + batch = sorted(orgs) + written = 0 + for i in range(0, len(batch), 1000): + status = await pg.execute( + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) + written += int(status.split()[-1]) + return written + + +async def read_mode(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The cell's state (`cell.read_state`) and its `first` row.""" + state = await cell.read_state(conn) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + return state, None if first is None else (bool(first["is_first"]), str(first["cell"])) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 8ad83098a9..cf83fa8f27 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -1,7 +1,10 @@ """`loom cell prepare`: register this cell in the router's directory and write the rows its processes read (docs/architecture/cells.md, "Preparing a cell" and "The tenancy check"). -It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names): +It runs where the cell's stores are configured (`LOOM_CELL` set to the cell it names), +holding the attach lock (`cell.ATTACH_LOCK_KEY`) on a direct session of the cell's Postgres, +after that session's checks (one backend, the tenancy marker kept, a UTF8 database), so it +never runs beside an attach or another prepare: 1. It reads the cell's Postgres (deployment id, `first`, mode and own rows, tenancy rows) and the live processes' heartbeats, and refuses a cell whose rows name another cell @@ -23,11 +26,13 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets import sys import time +from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -46,6 +51,9 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound +# A bound on the lock session's statements: it reads the catalog and holds the lock. +_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads + _sleep = asyncio.sleep # the seam the retry tests replace @@ -442,6 +450,38 @@ async def prepare( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, +) -> int: + """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" + async with attach_lock(): + return await _prepare_locked(ch, args, router, transport) + + +@contextlib.asynccontextmanager +async def attach_lock() -> AsyncIterator[None]: + """The attach lock, held on a direct session of the cell's Postgres while the block + runs, after that session's checks: prepare and attach write the same probe nonce, and + one of them runs at a time.""" + from server.cli import cell_attach_pg + from server.cli.cell_ch import Refused as Stopped + from server.core import pg + + async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + try: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + except Stopped as e: + raise Refused(str(e)) from None + yield + + +async def _prepare_locked( + ch: Any, + args: argparse.Namespace, + router: httpx.URL, + transport: httpx.AsyncBaseTransport | None, ) -> int: rows = await _read_rows() if not rows.deployment_id: diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index 6968dd49b0..e7d9077a02 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -67,6 +67,19 @@ } ) +# Tables a worker drains into another table, deleting what it moved: each is scanned +# before every table that is not one, so a row moved mid-scan is read where it was or where +# it went. A table a materialized view reads is ordered the same way, from +# `system.tables` dependencies (`order_sources`). +MOVE_SOURCES = frozenset( + { + "loom.learnings_queue", + "loom.memory_code_link_queue", + "loom.memory_extraction_queue", + "loom.memory_webhook_outbox", + } +) + # A refreshable view's temporary table while a refresh exchanges its target. _EXCHANGE = re.compile(r"^\.tmp\.inner(_id)?\.") # A provider transition's copies (server/erasure/store/provider_copies.py). @@ -338,6 +351,32 @@ async def list_postgres(conn: Any) -> tuple[list[str], list[Refusal]]: return names, refusals +def order_sources(tables: list[Table], view_sources: Iterable[str]) -> list[Table]: + """`tables` with the move sources (`MOVE_SOURCES`, and every table a view reads, + `view_sources`, canonical names) first, each group by name.""" + first = MOVE_SOURCES | set(view_sources) + return sorted(tables, key=lambda t: (t.canonical not in first, t.canonical)) + + +async def view_sources( + ch: Any, topology: Topology, names: Mapping[str, str], dbs: list[str] +) -> set[str]: + """The tables of `dbs` a view depends on, as canonical names.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, database, name FROM {source:tables} " + "WHERE database IN {dbs:Array(String)} AND notEmpty(dependencies_table)", + {"dbs": dbs}, + ) + return { + f"{ch_names.logical(str(r['database']))}.{r['name']}" + for found in rows.values() + for r in found + } + + async def list_tables( ch: Any, pg_conn: Any, topology: Topology, names: Mapping[str, str] ) -> TableList: @@ -370,7 +409,8 @@ async def _list_tables( if copies := provider_copies(tables): out.refusals += copies return out - out.clickhouse, problems = judge_clickhouse(tables, classified, topology) + scanned, problems = judge_clickhouse(tables, classified, topology) + out.clickhouse = order_sources(scanned, await view_sources(ch, topology, names, databases())) out.refusals += problems out.refusals += await reverse_clickhouse(ch, tables, classified) out.postgres, problems = await list_postgres(pg_conn) @@ -460,7 +500,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def _pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -615,7 +655,7 @@ async def _read_scan( return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def _ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) @@ -697,10 +737,10 @@ async def scan( result.add("configured", org) with query_tag(TAG): for name in tables.postgres: - for org in await _pg_orgs(name, budget): + for org in await pg_orgs(name, budget): result.add(f"postgres {name}", org) for table in tables.clickhouse: - for raw in await _ch_orgs(ch, table, topology, budget): + for raw in await ch_orgs(ch, table, topology, budget): result.add(f"clickhouse {table.canonical}", raw) result.elapsed_s = budget.elapsed() return result diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..29a9ddabf6 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -566,10 +566,15 @@ def grade(p: Any) -> str: return [(f"problem:{p.code}", grade(p), f"{p.message} Fix: {p.fix}") for p in problems] -def _run_doctor_checks() -> list[tuple[str, str, str]]: +def _run_doctor_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The live checks (Python, settings, JWT, ClickHouse, LLM) followed - by the shared problem list, as (name, grade, detail).""" - return _run_live_checks() + _code_mirror_checks() + _problem_checks(_doctor_problems()) + by the shared problem list, as (name, grade, detail). `allowed_roles` is + `--pg-allow-role`, for a cell's checks.""" + return ( + _run_live_checks(allowed_roles) + + _code_mirror_checks() + + _problem_checks(_doctor_problems()) + ) def _code_mirror_checks() -> list[tuple[str, str, str]]: @@ -582,7 +587,7 @@ def _code_mirror_checks() -> list[tuple[str, str, str]]: return [("code_mirrors", WARN, f"not checked: {type(e).__name__}: {e}")] -def _run_live_checks() -> list[tuple[str, str, str]]: +def _run_live_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """Run the doctor's live checks and return them as (name, grade, detail) with grade one of PASS / WARN / FAIL. @@ -664,7 +669,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: if settings_ok and ch_ok and settings.cell: from server.cli import doctor_cell - checks += doctor_cell.cell_checks() + checks += doctor_cell.cell_checks(allowed_roles) # 5. LLM provider resolution: what extraction runs on and # what decided it; WARN when the provider was inferred (or named) but its @@ -677,6 +682,17 @@ def _run_live_checks() -> list[tuple[str, str, str]]: return checks +def _stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`--stray-orgs`, on a cell only (LOOM_CELL set).""" + from server.settings import settings + + if not settings.cell: + return [("cell_stray_orgs", WARN, "not read: LOOM_CELL is unset (no cell)")] + from server.cli import doctor_cell + + return doctor_cell.stray_org_checks(budget_s) + + def _client_checks(args: argparse.Namespace) -> list[tuple[str, str, str]]: """`loom doctor --client`: the checks from the machine a coding agent runs on, with the CLI's own HTTP helpers. A crash in one of @@ -722,7 +738,9 @@ def _cmd_doctor(_args: argparse.Namespace) -> int: operator sees the full picture. Exit 0 = nothing failed (warnings are listed but do not fail); 1 = at least one FAIL. """ - checks = _run_doctor_checks() + checks = _run_doctor_checks(tuple(getattr(_args, "pg_allow_role", None) or ())) + if getattr(_args, "stray_orgs", False): + checks += _stray_org_checks(_args.stray_orgs_budget) # The client's side of the same picture, when asked. client = _client_checks(_args) if getattr(_args, "client", False) else [] checks = _client_machine_grades(checks, client) + client @@ -841,4 +859,21 @@ def _add_doctor_flags(p_doc: argparse.ArgumentParser) -> None: ), ) p_doc.add_argument("--server-url", default=None, help="Override the client's server URL.") + p_doc.add_argument( + "--pg-allow-role", + action="append", + default=[], + help="On a cell: a Postgres role that may make an org held (the servers' role).", + ) + p_doc.add_argument( + "--stray-orgs", + action="store_true", + help="On a cell: also read the orgs its stores hold without a tenancy row.", + ) + p_doc.add_argument( + "--stray-orgs-budget", + type=float, + default=60.0, + help="Seconds --stray-orgs may read for, smallest tables first (default 60).", + ) p_doc.add_argument("--token", default=None, help="Override the client's token.") diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..c79723bbbc 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -1,7 +1,11 @@ """`loom doctor`'s cell checks, run only with LOOM_CELL set (docs/architecture/cells.md): the deployment check, a partial cell state, the tenancy marker on this process's -sessions and on every session, and the defects of the org triggers and the `loom.tenancy` -guard (`cell_triggers.missing`). +sessions and on every session, the defects of the org triggers and the `loom.tenancy` +guard (`cell_triggers.missing`), and the point-in-time reads attach makes, run again on +every doctor run: the login roles that can make an org held or lower the mode, and the +role and database settings and subscriptions that pass an unchecked writer +(`cell_attach_pg`). `--stray-orgs` also reads the orgs the cell's stores hold without a +tenancy row (`stray_org_checks`). A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; @@ -78,14 +82,34 @@ async def _unmarked_session() -> bool: """Whether a session that sends no marker passes the triggers all the same (a role, database, ALTER SYSTEM or options setting): marked, or in the replica role that fires no origin trigger.""" + from server.cli import cell_attach_pg + + return await cell_attach_pg.session_passes() + + +async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, str]]: + """Attach's point-in-time Postgres reads, again: the login roles outside the owner + roles, the superusers and `--pg-allow-role` that can make an org held or lower the + mode, and the settings and subscriptions that pass an unchecked writer.""" + from server.cli import cell_attach_pg from server.core import pg - async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS, marked=False) as conn: - value: str | None = await conn.fetchval( - "SELECT current_setting($1, true)", pg.TENANCY_MARKER - ) - role: str = await conn.fetchval("SELECT current_setting('session_replication_role')") - return value == "on" or role == "replica" + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + holders = await cell_attach_pg.privilege_holders(conn) + unchecked = await cell_attach_pg.unmarked_refusals(conn, False) + roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) + return [ + ( + "cell_tenancy_privilege", + WARN if roles else PASS, + "; ".join(str(r) for r in roles) or "no role outside the owner roles can", + ), + ( + "cell_unchecked_writers", + WARN if unchecked else PASS, + "; ".join(str(r) for r in unchecked) or "no setting or subscription passes one", + ), + ] def _grades( @@ -149,7 +173,7 @@ def _grades( return out -async def _checks(ch: Any) -> list[tuple[str, str, str]]: +async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): return [("cell_state", PASS, f"cell checks not run: {before}")] @@ -159,7 +183,11 @@ async def _checks(ch: Any) -> list[tuple[str, str, str]]: return [("cell_deployment", FAIL, f"deployment check refused: {e}")] except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + try: + reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) + except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run + reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + return _grades(state, marked, marks_all, defects) + reads def _reason(e: BaseException) -> str: @@ -167,9 +195,10 @@ def _reason(e: BaseException) -> str: return f"{type(e).__name__}: {str(e) or 'no answer within the bound'}" -def cell_checks() -> list[tuple[str, str, str]]: +def cell_checks(allowed_roles: tuple[str, ...] = ()) -> list[tuple[str, str, str]]: """The cell checks, on the CLI's ClickHouse client connected for them alone (no CLI - deployment check: a deployment not booted yet is named, not refused).""" + deployment check: a deployment not booted yet is named, not refused). `allowed_roles` + is doctor's `--pg-allow-role`.""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client @@ -180,10 +209,121 @@ async def _go() -> list[tuple[str, str, str]]: connect_declared(ch, org_free=reads("loom doctor reads a cell's state")), timeout=_SESSION_S, ) - return await _checks(ch) + return await _checks(ch, allowed_roles) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: await disconnect(ch) return asyncio.run(_go()) + + +# -- --stray-orgs -------------------------------------------------------------------------- + + +async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: + """Each ClickHouse table's active row count, from `system.parts` (KeeperMap: 0).""" + from server.core import ch_names + + rows = await ch.query_all( + "SELECT database, table, sum(rows) AS n FROM system.parts WHERE active " + "AND database IN {dbs:Array(String)} GROUP BY database, table", + {"dbs": sorted({t.database for t in tables})}, + ) + return {f"{ch_names.logical(str(r['database']))}.{r['table']}": int(r["n"]) for r in rows} + + +async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: + from server.cli import cell_scan + from server.cli.cell_ch import Refused, databases, read_topology + from server.core import pg + from server.core.cell_triggers import ORG_TABLES + + topology = await read_topology(ch) + if topology.multi: + from server.cli.cell_writers import host_names + + names = await host_names(ch, topology) + else: + names = {} + listed, problems = await cell_scan.list_clickhouse(ch, topology, names, databases()) + ch_tables = [t for t in listed if t.engine.endswith("MergeTree") or t.engine == "KeeperMap"] + sizes = await _table_sizes(ch, ch_tables) + async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: + held = {str(r["org"]) for r in await conn.fetch("SELECT org FROM loom.tenancy")} + pg_rows = await conn.fetch( + "SELECT 'loom.' || c.relname AS name, greatest(c.reltuples, 0)::bigint AS n " + "FROM pg_class c WHERE c.oid IN (SELECT to_regclass('loom.' || t.relname) FROM (" + + ORG_TABLES + + ") t)" + ) + # Smallest first: a budget that runs out leaves the largest unchecked. + work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] + work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] + work.sort(key=lambda w: (w[0], str(w[2]))) + budget = cell_scan.Budget(budget_s) + stray: dict[str, list[str]] = {} + unchecked: list[str] = [] + for i, (_n, store, table) in enumerate(work): + found = cell_scan.ScanResult() + name = table if store == "postgres" else table.canonical + try: + if store == "postgres": + for org in await cell_scan.pg_orgs(table, budget): + found.add(name, org) + else: + for raw in await cell_scan.ch_orgs(ch, table, topology, budget): + found.add(name, raw) + except Refused: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) + bad = [m for ms in found.malformed.values() for m in ms] + if orgs or bad: + log = " (a log table: never placed)" if name in cell_scan.LOG_TABLES else "" + stray[f"{store} {name}{log}"] = orgs[:5] + bad[:5] + out = [ + ( + "cell_stray_orgs", + WARN if stray else PASS, + "; ".join(f"{t}: {', '.join(o)}" for t, o in sorted(stray.items())) + or "every org found has a tenancy row", + ) + ] + if unchecked or problems: + out.append( + ( + "cell_stray_orgs_unchecked", + WARN, + f"not read within --stray-orgs-budget ({budget_s:g} s): " + + ", ".join(unchecked + [str(p) for p in problems]), + ) + ) + return out + + +def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: + """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row + (log-only orgs, names no request can carry, rows an unchecked writer left), smallest + tables first within one budget, naming every table left unchecked. Never a FAIL: the + stray-org erase command removes what it names.""" + from server.cli._ch import connect_declared, disconnect + from server.core.ch import get_client, statement_timeout + + async def _go() -> list[tuple[str, str, str]]: + ch = get_client() + try: + await asyncio.wait_for( + connect_declared(ch, org_free=reads("loom doctor reads a cell's orgs")), + timeout=_SESSION_S, + ) + if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): + return [("cell_stray_orgs", PASS, f"not read: {before}")] + return await _stray_reads(ch, budget_s) + except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run + return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] + finally: + await disconnect(ch) + + with statement_timeout(budget_s): + return asyncio.run(_go()) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 19cf492b0e..9c2c9755ae 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -36,7 +36,9 @@ CELL_FIRST_TRIGGERS = ("cell_first_immutable", "cell_first_no_truncate") # Org-keyed tables the org triggers skip: the cell's own tenancy rows (`tenancy_record` # guards them instead), and the idempotency rows keyed by the caller's token org (a platform -# admin's replay row for its home org is neither refused nor placed). +# admin's replay row for its home org is not refused). Outside the guard is not outside the +# first attach: its org scan (`server/cli/cell_scan.py`) reads every table with an `org` +# column, these included, and places the orgs it finds. EXCLUDED = frozenset({"tenancy", "idempotency", "idempotency_fences"}) # A table's DDL waits this long for its lock a try, LOCK_ATTEMPTS tries LOCK_RETRY_S # apart: a boot never queues behind a long transaction (and every write of the table @@ -77,10 +79,13 @@ # What a trigger's DDL (SHARE ROW EXCLUSIVE) waited behind on a table, read after the # wait failed (pg_blocking_pids is empty by then): the locks held in a mode it conflicts # with (every mode but the two readers take), and the requests queued ahead of it in -# one, whose own blocker heads the queue. +# one, whose own blocker heads the queue. An anti-wraparound autovacuum, which never yields +# its lock, says so in its activity line (compared here, never returned). _HOLDERS = """ SELECT DISTINCT l.pid, l.granted, a.backend_type, a.state, a.application_name, - a.xact_start, a.wait_event + a.xact_start, a.wait_event, + coalesce(a.backend_type = 'autovacuum worker' + AND a.query LIKE '%(to prevent wraparound)', false) AS wraparound FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = $1::regclass AND l.pid <> pg_backend_pid() AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) @@ -208,63 +213,107 @@ async def ensure(conn: Any) -> None: tries; a table whose lock it cannot take is logged with the holders of its locks, one the role cannot alter is logged as such, and either is skipped (the boot serves). A failed read of the state or the catalog raises. Runs on the migration session, which - holds the migration lock.""" + holds the migration lock. `ensure_table` runs one table's part alone.""" state = await cell.read_state(conn) if state == "open": return - # A table's statements, in one transaction: one lock wait, and every statement or none. - statements: dict[str, list[str]] = {} first = await _triggers(conn, CELL_FIRST_TRIGGERS) - for name in CELL_FIRST_TRIGGERS: - row = first.get(("cell_first", name)) - if row is not None and row["enabled"] != "A": - statements.setdefault("cell_first", []).append( - f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + always = [ + f"ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER {name}" + for name in CELL_FIRST_TRIGGERS + if (row := first.get(("cell_first", name))) is not None and row["enabled"] != "A" + ] + if always: + await _run_ddl(conn, "cell_first", always) + if state != "enforced": + return + tables = await conn.fetch(ORG_TABLES) + for r in tables: + if r["relkind"] == "p": + # No trigger guards it: named here as missing() names it. + log.warning( + "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] ) - if state == "enforced": - present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) - tables = await conn.fetch(ORG_TABLES) - for r in tables: - if r["relkind"] == "p": - # No trigger guards it: named here as missing() names it. - log.warning( - "cell triggers: loom.%s is partitioned: unsupported, unguarded", r["relname"] - ) - for r, names in _planned(tables): - table = r["relname"] - for name in names: - row = present.get((table, name)) - if row is None: - statements.setdefault(table, []).append(_create(f"loom.{_ident(table)}", name)) - elif row["enabled"] != "O": - statements.setdefault(table, []).append( - f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}" - ) - for table, sqls in statements.items(): - await _run_ddl(conn, table, sqls) + present = await _triggers(conn, (*ORG_TRIGGERS, TENANCY_TRIGGER)) + for r, names in _planned(tables): + if sqls := _table_statements(r["relname"], names, present): + await _run_ddl(conn, r["relname"], sqls) + + +async def planned_tables(conn: Any) -> list[str]: + """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" + return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + + +async def ensure_table( + conn: Any, + table: str, + *, + recording_first: bool = False, + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its + org triggers), so one table's DDL can run alone: in `enforced`, or with + `recording_first` also in a first cell's `recording`, where the org triggers' function + passes every row and a boot never creates them. False when the DDL was not applied (its + lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); + True when it was, or nothing was needed. Raises for a table outside the planned set, or + in another state.""" + state = await cell.read_state(conn) + if state != "enforced" and not (recording_first and state == "recording"): + raise RuntimeError(f"cell triggers: not created in state {state!r}") + planned = [ + (r, names) for r, names in _planned(await conn.fetch(ORG_TABLES)) if r["relname"] == table + ] + if not planned: + raise ValueError(f"loom.{table} carries no cell triggers") + ((_, names),) = planned + sqls = _table_statements(table, names, await _triggers(conn, names)) + return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + + +def _table_statements( + table: str, names: tuple[str, ...], present: dict[tuple[str, str], Any] +) -> list[str]: + """The DDL that gives `table` its named triggers, each present and enabled.""" + out = [] + for name in names: + row = present.get((table, name)) + if row is None: + out.append(_create(f"loom.{_ident(table)}", name)) + elif row["enabled"] != "O": + out.append(f"ALTER TABLE loom.{_ident(table)} ENABLE TRIGGER {name}") + return out -async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries. The boot serves past - the two causes cells.md names (its lock not taken, a table this role cannot alter); - any other error fails it, as a failed read of the state does.""" +async def _run_ddl( + conn: Any, + table: str, + sqls: list[str], + lock_timeout: str = LOCK_TIMEOUT, + attempts: int = LOCK_ATTEMPTS, +) -> bool: + """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The + boot serves past the two causes cells.md names (its lock not taken, a table this role + cannot alter); any other error fails it, as a failed read of the state does.""" import asyncpg - for attempt in range(LOCK_ATTEMPTS): + for attempt in range(attempts): try: async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'") + await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return + return True except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return + return False except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. - if attempt < LOCK_ATTEMPTS - 1: + if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue log.warning( @@ -274,13 +323,19 @@ async def _run_ddl(conn: Any, table: str, sqls: list[str]) -> None: "; ".join(sqls), await _holders(conn, table), ) + return False + + +async def holders(conn: Any, table: str) -> list[Any]: + """The sessions a blocked DDL on `table` waits behind (`_HOLDERS`).""" + return list(await conn.fetch(_HOLDERS, f"loom.{_ident(table)}")) async def _holders(conn: Any, table: str) -> str: """The sessions a blocked DDL waited behind, never their query text (a literal in it may be memory content).""" try: - rows = await conn.fetch(_HOLDERS, f"loom.{_ident(table)}") + rows = await holders(conn, table) except Exception as e: # noqa: BLE001 — naming the holders is best effort log.warning("cell triggers: the lock holders of loom.%s not read: %s", table, e) return "lock holders not read" diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..bb47bfd945 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -53,6 +53,10 @@ _LIVE_INTERVALS = 3 # How often the leader drops the KeeperMap keys of processes gone an hour. _PRUNE_EVERY_S = 600.0 +# How old a row's last beat is before the leader drops its key. A contract across builds, +# not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader +# that sweeps them may run another build, so a change is a design change of its own. +PRUNE_AFTER_S = 3600 # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -130,6 +134,22 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + +def boot_allowance_s() -> float: + """How long a booting process takes to report its first cell row and its first router + answer: two beats and one router read.""" + return 2 * float(settings.instance_heartbeat_seconds) + float(settings.cell_router_timeout_s) + + async def beat(ch: Any, *, stopped: bool = False) -> None: """Write this process's row. A KeeperMap insert replaces the key.""" await ch.insert(TABLE, [row(stopped=stopped)]) @@ -197,6 +217,34 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out +async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose widened window + (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or + past now without one: stopped rows included, which `read_live` drops. `stopped` is its + newest row's flag and `live` whether that row is inside the live window and not + stopped, both judged in ClickHouse.""" + rows = await ch.query_all( + f""" + SELECT instance_id, + argMax(keys_fp, heartbeat_at) AS keys_fp, + argMax(stopped, heartbeat_at) AS stopped, + toString(max(heartbeat_at)) AS last_beat, + max(heartbeat_at) > now64(3) - toIntervalMillisecond({{live:UInt32}}) + AND stopped = 0 AS live + FROM {TABLE} + GROUP BY instance_id + HAVING max(heartbeat_at) + toIntervalMillisecond({{wide:UInt32}}) + > if({{since:String}} = '', now64(3), toDateTime64({{since:String}}, 3)) + """, + params={ + "live": int(live_window_s() * 1000), + "wide": int(widened_window_s() * 1000), + "since": since or "", + }, + ) + return [dict(r) for r in rows] + + def live_rows() -> list[dict[str, Any]]: """The live rows as of the last read; none before the first read or once that read is older than the live window (ClickHouse unreachable).""" @@ -224,14 +272,15 @@ async def _prune(ch: Any) -> None: # Each table's sweep stands alone: one failing leaves the other to run. try: await ch.execute( - "ALTER TABLE loom.instances DELETE WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + "ALTER TABLE loom.instances DELETE " + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("instance row sweep failed: %s", e) try: await ch.execute( "ALTER TABLE loom.cell_heartbeats DELETE " - "WHERE heartbeat_at < now64(3) - toIntervalHour(1)" + f"WHERE heartbeat_at < now64(3) - toIntervalSecond({PRUNE_AFTER_S})" ) except Exception as e: # noqa: BLE001 — a later sweep retries log.warning("cell heartbeat sweep failed: %s", e) @@ -331,8 +380,10 @@ def _reset_for_tests() -> None: __all__ = [ "CELL_TABLE", + "PRUNE_AFTER_S", "TABLE", "beat", + "boot_allowance_s", "cell_beat", "cell_row", "instance_id", @@ -341,8 +392,10 @@ def _reset_for_tests() -> None: "live_window_s", "publish_config", "read_live", + "read_recent", "schema_version", "start", "stop", "tick", + "widened_window_s", ] diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py new file mode 100644 index 0000000000..406d277646 --- /dev/null +++ b/tests/integration/test_cell_attach.py @@ -0,0 +1,635 @@ +"""`loom cell attach` (server/cli/cell_attach.py) against a real Postgres with a synchronous +standby, a real ClickHouse and the router served in process: a prepared first cell in +`recording` with orgs in both stores attaches to `serving` with every org placed and the +triggers created; a rerun on `serving` exits 0; a refusal after step 3 leaves `enforced` and +a rerun resumes from there; a second run is refused at the attach lock; the Postgres reads +attach makes refuse what they name (an unchecked writer's setting, a subscription, a role +that can make an org held, a transaction that may have read `recording`). + +The cell's Postgres and the directory are databases of the test's own; the configured +ClickHouse databases are scratch ones; the cell's processes are heartbeat rows the test +writes. Needs LOOM_PG_DSN (a server with a synchronous standby), LOOM_ROUTER_PG_DSN and +LOOM_CH_HOST.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell as cell_cli +from server.cli import cell_attach, cell_attach_pg, cell_prepare +from server.cli.cell_ch import Refused +from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core.ch import get_client +from server.core.schema import pg_migrate +from server.router import app as router_app +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +OPERATOR = "lco_" + "2" * 128 +CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() +ROUTER = "http://127.0.0.1:7901" +HOST = "loom.example" + + +class _Network(httpx.AsyncBaseTransport): + """What attach and the router reach: the router at ROUTER, the shared host's health + route, the cell's upstream main port (:7777) and its probe listener (any other).""" + + def __init__(self) -> None: + self.listener = httpx.ASGITransport(app=cell_probe.listener_app()) + self.router = httpx.ASGITransport( + app=router_app.build_app(router_app.Router(probe_transport=self)) + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "127.0.0.1" and request.url.port == 7901: + return await self.router.handle_async_request(request) + if request.url.host == HOST or request.url.port == 7777: + return httpx.Response(200, json=cell_probe.answer()) + return await self.listener.handle_async_request(request) + + +async def _admin(sql: str, *args: Any) -> Any: + import asyncpg + + conn = await asyncpg.connect(os.environ["LOOM_PG_DSN"]) + try: + return await conn.execute(sql, *args) + finally: + await conn.close() + + +@pytest.fixture +async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: + cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") + cell_db, own_cell = await create_database(cell_dsn) + router_db, own_router = await create_database(router_dsn) + deployment = f"dep-{uuid.uuid4().hex[:8]}" + tag = f"it-{uuid.uuid4().hex[:8]}-" + for name, value in { + "cell": "first", + "storage_authority": "postgres", + "pg_dsn": own_cell, + "document_s3_bucket": "", + "router_pg_dsn": own_router, + "router_shared_host": HOST, + "router_upstream_probe_url": "http://listener.internal:7790", + "router_upstream_url": "http://upstream.internal:7777", + "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), + "router_operator_token": OPERATOR, + "ch_cluster": "", + "public_base_url": f"https://{HOST}", + "oauth_issuer": "", + }.items(): + monkeypatch.setattr(settings, name, value) + await pg.close_pool() + monkeypatch.setattr(directory, "_pool", None) + # A step's poll interval and the waits stay short. + monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) + ch = get_client() + await ch.connect() + ids: list[str] = [] + try: + await pg_migrate.migrate() + await pg.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) " + "VALUES ('postgres', 1, $1)", + deployment, + ) + await directory.migrate() + await ch.insert( + "loom.storage_authority", + [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + ) + cell.set_deployment_id(deployment) + yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} + finally: + cell.set_deployment_id("") + await ch.execute( + "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " + "SETTINGS mutations_sync = 2", + {"d": deployment}, + ) + for table in (instances.TABLE, instances.CELL_TABLE): + await ch.execute( + f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", + {"ids": ids}, + ) + await ch.execute( + "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} + ) + await ch.disconnect() + await pg.close_pool() + await directory.close() + await drop_database(cell_dsn, cell_db) + await drop_database(router_dsn, router_db) + + +def _value(**keys: Any) -> str: + reported = { + **cell.as_value(cell.mine()), + "serves": True, + "marked": True, + "router": True, + "router_why": "", + **keys, + } + return json.dumps(reported, sort_keys=True) + + +async def _beat( + stores: dict[str, Any], + instance_id: str, + *, + cell_name: str | None = "first", + mode: str = "recording", + stopped: bool = False, + **keys: Any, +) -> None: + """One process's heartbeat rows, as `instances.tick` writes them.""" + ch = stores["ch"] + if instance_id not in stores["ids"]: + stores["ids"].append(instance_id) + await ch.insert( + instances.TABLE, [{**instances.row(stopped=stopped), "instance_id": instance_id}] + ) + if cell_name is not None: + await ch.insert( + instances.CELL_TABLE, + [ + { + "instance_id": instance_id, + "cell": cell_name, + "tenancy_mode": mode, + "fingerprint": _value(**keys), + } + ], + ) + + +async def _prepare(stores: dict[str, Any], network: _Network) -> None: + pid = f"it-{uuid.uuid4().hex[:8]}" + await _beat(stores, pid, cell_name="first", mode="open") + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + assert await cell_prepare.prepare(stores["ch"], args, router, network) == 0 + stores["pid"] = pid + await _beat(stores, pid, mode="recording") + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 20, + "scan_budget": 600, + "dry_run": False, + "ch_allow_user": [settings.ch_user or "default"], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": ROUTER, + } + return argparse.Namespace(**{**fields, **over}) + + +async def _attach(stores: dict[str, Any], network: _Network, **over: Any) -> int: + args = _args(**over) + url = cell_cli.operator_router(args) + assert isinstance(url, httpx.URL) + return await cell_attach.attach(stores["ch"], args, url, network) + + +async def _mode() -> str: + return str(await pg.fetchval("SELECT mode FROM loom.tenancy_mode")) + + +async def _placed() -> set[str]: + import asyncpg + + conn = await asyncpg.connect(settings.router_pg_dsn) + try: + rows = await conn.fetch("SELECT org FROM loom_router.placements WHERE cell = 'first'") + finally: + await conn.close() + return {r["org"] for r in rows} + + +async def _row() -> Any: + return await directory.first_cell_named("first") + + +@pytest.mark.asyncio +async def test_a_recording_first_cell_attaches_to_serving(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + tag = stores["tag"] + await _prepare(stores, network) + capsys.readouterr() + # Orgs a recording cell holds: a tenancy row alone, a Postgres data row, a ClickHouse + # row, and one name no request can carry (skipped and named). + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "signed-in") + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "pg") + await stores["ch"].insert( + "loom.namespaces", + [ + {"org": tag + "ch", "namespace": "n", "owner_email": ""}, + {"org": tag + "bad\x00org", "namespace": "n", "owner_email": ""}, + ], + ) + assert await _attach(stores, network) == 0 + report = json.loads(capsys.readouterr().out) + mine = {tag + "signed-in", tag + "pg", tag + "ch"} + assert report["state"] == "serving" + assert await _mode() == "enforced" + placed = await _placed() + assert {o for o in placed if o.startswith(tag)} == mine + held = {r["org"] for r in await pg.fetch("SELECT org FROM loom.tenancy")} + assert held == placed + assert any("clickhouse loom.namespaces" in k for k in report["skipped"]["malformed"]) + row = await _row() + assert (row.state, row.mode, row.placed) == ("serving", "enforced", len(placed)) + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + # A rerun on `serving` is done. + assert await _attach(stores, network) == 0 + assert json.loads(capsys.readouterr().out)["done"] is True + + +@pytest.mark.asyncio +async def test_a_refusal_after_step_3_leaves_enforced_and_a_rerun_resumes( + cell_stores, monkeypatch, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "a" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + # A process with LOOM_CELL unset appears once the mode is committed: refused at once. + real = cell_attach.Run.durable + + async def durable_then_unset(self: Any) -> int: + xmax = await real(self) + await _beat(stores, f"it-unset-{uuid.uuid4().hex[:6]}", cell_name="") + return xmax + + monkeypatch.setattr(cell_attach.Run, "durable", durable_then_unset) + assert await _attach(stores, network) == 1 + assert "runs with LOOM_CELL unset" in capsys.readouterr().err + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("attaching", "enforced") + # Placed live once enforced; the rerun resumes from the durable read. + monkeypatch.setattr(cell_attach.Run, "durable", real) + for i in list(stores["ids"]): + if i.startswith("it-unset-"): + await _beat(stores, i, cell_name="first", mode="enforced") + await _beat(stores, stores["pid"], mode="enforced") + assert await _attach(stores, network) == 0 + assert org in await _placed() + assert (await _row()).state == "serving" + + +@pytest.mark.asyncio +async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "the test") + assert await _attach(stores, network) == 1 + assert "attach lock: held by pid" in capsys.readouterr().err + assert await _mode() == "recording" + + +@pytest.mark.asyncio +async def test_a_dry_run_writes_nothing(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + capsys.readouterr() + org = stores["tag"] + "dry" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + assert await _attach(stores, network, dry_run=True) == 0 + report = json.loads(capsys.readouterr().out) + assert report["per_source"]["postgres loom.scope_generations"] == { + "count": 1, + "sample": [org], + } + assert await _mode() == "recording" + assert await _placed() == set() + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with pg.dedicated() as conn: + assert any(d.kind == "absent" for d in await cell_triggers.missing(conn)) + + +@pytest.mark.asyncio +async def test_an_org_past_the_limit_refuses_before_any_tenancy_row(cell_stores, capsys) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + long = "o" * (cell.ORG_MAX_BYTES + 1) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", long) + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('org-ok', '*')") + assert await _attach(stores, network) == 1 + assert f"longer than {cell.ORG_MAX_BYTES} bytes" in capsys.readouterr().err + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert (await _row()).placed == 0 + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_that_pass_unchecked_writers_refuse( + cell_stores, capsys +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + db = stores["db"] + role = f"it_role_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + # Stored as first spelled: a fresh session that never set the placeholder. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" SET "Loom.Tenancy_Checked" = \'on\'') + async with pg.dedicated() as conn: + setting = await conn.fetchval( + "SELECT setconfig::text FROM pg_db_role_setting s JOIN pg_roles r " + "ON r.oid = s.setrole WHERE r.rolname = $1", + role, + ) + assert "Loom.Tenancy_Checked" in setting + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [r.subject for r in refusals] == [f"postgres setting for {role} in {db}"] + # Another database's entry does not refuse. + await _admin(f'ALTER ROLE {role} IN DATABASE "{db}" RESET ALL') + await _admin( + f"ALTER ROLE {role} IN DATABASE postgres SET session_replication_role = 'replica'" + ) + async with pg.dedicated() as conn: + assert await cell_attach_pg.unmarked_refusals(conn, False) == [] + # A role that can insert a tenancy row, a column grant alone, refuses attach. + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + assert await _attach(stores, network) == 1 + err = capsys.readouterr().err + assert f"postgres role {role}: may insert into loom.tenancy" in err + assert await _mode() == "recording" + assert await _attach(stores, network, pg_allow_role=[role], dry_run=True) == 0 + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} IN DATABASE postgres RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_disabled_subscription_refuses(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + name = f"it_sub_{uuid.uuid4().hex[:6]}" + await pg.execute( + f"CREATE SUBSCRIPTION {name} CONNECTION 'host=127.0.0.1 dbname=none' " + "PUBLICATION none WITH (connect = false, enabled = false)" + ) + try: + async with pg.dedicated() as conn: + refusals = await cell_attach_pg.unmarked_refusals(conn, False) + assert [str(r) for r in refusals] == [ + f"postgres subscription {name}: disabled: its apply worker fires no origin " + "trigger; drop it" + ] + finally: + await pg.execute(f"ALTER SUBSCRIPTION {name} SET (slot_name = NONE)") + await pg.execute(f"DROP SUBSCRIPTION {name}") + + +@pytest.mark.asyncio +async def test_step_4_waits_for_a_transaction_that_may_have_read_recording(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as writer, pg.dedicated() as poller: + tx = writer.transaction() + await tx.start() + await writer.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ('w', '*')") + # X is read after a later transaction completed, as after the durable read. + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + # A session idle outside a transaction holds nothing: the wait passes. + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + + +@pytest.mark.asyncio +async def test_the_session_checks_and_the_encoding(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as conn: + assert await cell_attach_pg.direct_refusals(conn, "attach") == [] + assert await cell_attach_pg.encoding_refusals(conn) == [] + async with pg.dedicated(marked=False) as conn: + refusals = await cell_attach_pg.direct_refusals(conn, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + f"attach's session loses {pg.TENANCY_MARKER}" + ] + name = f"it_latin_{uuid.uuid4().hex[:6]}" + await _admin( + f"CREATE DATABASE {name} ENCODING 'LATIN1' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0" + ) + try: + import asyncpg + + from tests.integration._pg_database import with_database + + conn = await asyncpg.connect(with_database(os.environ["LOOM_PG_DSN"], name)) + try: + (refusal,) = await cell_attach_pg.encoding_refusals(conn) + assert "LATIN1, not UTF8" in refusal.reason + finally: + await conn.close() + finally: + await _admin(f"DROP DATABASE {name}") + + +@pytest.mark.asyncio +async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + logged: list[str] = [] + + async def ok() -> bool: + return True + + async with pg.dedicated() as holder, pg.dedicated() as conn: + tx = holder.transaction() + await tx.start() + await holder.execute("SELECT mode FROM loom.tenancy_mode FOR SHARE") + + async def release() -> None: + await asyncio.sleep(3) + await tx.rollback() + + task = asyncio.create_task(release()) + await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await task + assert await _mode() == "enforced" + assert logged == ["the mode's lock is busy; retrying"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "grant,why", + [ + ("GRANT INSERT (org) ON loom.tenancy TO {r}", "may insert into loom.tenancy"), + ("GRANT UPDATE (org) ON loom.tenancy TO {r}", "may update loom.tenancy (org)"), + ("GRANT UPDATE (mode) ON loom.tenancy_mode TO {r}", "may update loom.tenancy_mode (mode)"), + ("GRANT INSERT (mode) ON loom.tenancy_mode TO {r}", "may insert into loom.tenancy_mode"), + ( + "GRANT INSERT, TRUNCATE ON loom.tenancy_mode TO {r}", + "may insert into loom.tenancy_mode", + ), + ( + "GRANT TRIGGER ON loom.scope_generations TO {r}", + "holds TRIGGER on loom.scope_generations", + ), + ("GRANT CREATE ON SCHEMA public TO {r}", "may create in schema public"), + ("GRANT CREATE ON SCHEMA loom TO {r}", "may create in schema loom"), + ( + "GRANT SET ON PARAMETER session_replication_role TO {r}", + "may set session_replication_role", + ), + ( + "SET loom.tenancy_checked = 'off'; " + "GRANT ALTER SYSTEM ON PARAMETER loom.tenancy_checked TO {r}", + "may ALTER SYSTEM SET loom.tenancy_checked", + ), + ("GRANT pg_write_server_files TO {r}", "is a member of pg_write_server_files"), + # A member of the owner role, ADMIN only: it can grant itself the rest. + ("GRANT {owner} TO {r} WITH ADMIN TRUE, INHERIT FALSE, SET FALSE", "owns loom.tenancy"), + ], +) +async def test_each_right_that_makes_an_org_held_is_named( + cell_stores, grant: str, why: str +) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_r_{uuid.uuid4().hex[:6]}" + owner = await pg.fetchval("SELECT current_user") + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) + for sql in grant.format(r=role, owner=owner).split("; "): + await conn.execute(sql) + holders = await cell_attach_pg.privilege_holders(conn) + assert owner in holders.own + assert any(why in w for w in holders.others.get(role, [])), holders.others + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_f_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") + holders = await cell_attach_pg.privilege_holders(conn) + await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") + assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] + finally: + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_a_repeatable_read_snapshot_from_before_x_holds_step_4(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + async with pg.dedicated() as reader, pg.dedicated() as poller: + tx = reader.transaction(isolation="repeatable_read") + await tx.start() + await reader.fetchval("SELECT mode FROM loom.tenancy_mode") # its snapshot, no xid + await poller.execute("BEGIN; SELECT pg_current_xact_id(); COMMIT") + xmax = await cell_attach_pg.snapshot_xmax(poller) + with pytest.raises(Refused, match="may have read `recording`"): + await cell_attach_pg.wait_transactions(poller, xmax, cell_attach_pg.Wait(0.5)) + await tx.rollback() + + +@pytest.mark.asyncio +async def test_prepare_is_refused_while_attach_holds_the_lock(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + args = argparse.Namespace( + cell="first", + host=HOST, + credential_sha256=CREDENTIAL_SHA, + probe_url="", + wait=0, + router=ROUTER, + ) + router = cell_cli.operator_router(args) + assert isinstance(router, httpx.URL) + async with pg.dedicated() as holder: + await cell_attach_pg.take_lock(holder, "attach") + with pytest.raises(cell_prepare.Refused, match="attach lock: held by pid"): + await cell_prepare.prepare(cell_stores["ch"], args, router, _Network()) + + +@pytest.mark.asyncio +async def test_doctor_names_the_same_holders_settings_and_subscriptions(cell_stores) -> None: + from server.cli import doctor_cell + + await _prepare(cell_stores, _Network()) + role = f"it_d_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + await pg.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await pg.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await _admin(f"ALTER ROLE {role} SET session_replication_role = 'replica'") + lines = {n: (g, d) for n, g, d in await doctor_cell._writer_reads(())} + assert lines["cell_tenancy_privilege"][0] == "WARN" + assert f"postgres role {role}" in lines["cell_tenancy_privilege"][1] + assert lines["cell_unchecked_writers"][0] == "WARN" + allowed = {n: g for n, g, _d in await doctor_cell._writer_reads((role,))} + assert allowed["cell_tenancy_privilege"] == "PASS" + finally: + await pg.execute(f"REVOKE ALL ON loom.tenancy FROM {role}") + await pg.execute(f"REVOKE ALL ON SCHEMA loom FROM {role}") + await _admin(f"ALTER ROLE {role} RESET ALL") + await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", tag + "held") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "held" + ) + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + grade, detail = lines["cell_stray_orgs"] + assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail + tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} + assert tight["cell_stray_orgs_unchecked"][0] == "WARN" diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index d1892be846..c78bc40331 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -166,7 +166,7 @@ def walk(node: dict[str, Any]) -> None: ("Index Only Scan", "tenancy_pkey"), ("Index Only Scan", "tenancy_pkey"), ], scans - assert len(await cell_scan._pg_orgs("loom.tenancy", Budget(60))) == 20000 + assert len(await cell_scan.pg_orgs("loom.tenancy", Budget(60))) == 20000 # Every org-keyed table of the schema has the index the scan needs. names, refusals = await cell_scan.list_postgres(own_pg) assert refusals == [] diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py new file mode 100644 index 0000000000..015cebd948 --- /dev/null +++ b/tests/unit/test_cell_attach.py @@ -0,0 +1,682 @@ +"""`loom cell attach` (server/cli/cell_attach.py, cell_attach_beats.py, cell_attach_pg.py) on +fakes: every refusal the heartbeats, the Postgres reads and the router's answers make, the +waits and their budget, the placement batches and the rescan's watch. The real Postgres, +ClickHouse and router behaviour is tests/integration/test_cell_attach.py. CH-free.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import httpx +import pytest + +from server.cli import cell_attach, cell_scan +from server.cli import cell_attach_beats as beats +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal, Refused +from server.cli.cell_scan import Table +from server.core import cell, instances +from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES +from server.settings import settings + +FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} +CURRENT = cell.as_value(FIELDS) +NEXT = cell.as_value({**FIELDS, "oidc_client_id": "next"}) + + +def _value(**keys: Any) -> dict[str, Any]: + return {**CURRENT, "serves": True, "marked": True, "router": True, "router_why": "", **keys} + + +def _beat( + i: str = "p1", + *, + cell_name: str | None = "first", + live: bool = True, + stopped: bool = False, + mode: str = "recording", + value: dict[str, Any] | None = None, + keys_fp: str = "k", +) -> beats.Beat: + return beats.Beat( + instance_id=i, + live=live, + stopped=stopped, + keys_fp=keys_fp, + last_beat="2026-01-01 00:00:00.000", + cell=cell_name, + mode=mode if cell_name is not None else "", + value=(value if value is not None else _value()) if cell_name is not None else None, + ) + + +def _judge(found: list[beats.Beat], stopped_unset: beats.StoppedUnset = "refuse") -> Any: + return beats.judge(found, "first", (CURRENT, NEXT), stopped_unset) + + +def _reasons(verdict: beats.Verdict) -> list[str]: + return [str(r) for r in verdict.refusals] + + +# -- the heartbeats -------------------------------------------------------------------- + + +def test_a_healthy_cell_passes_and_reports_its_keys() -> None: + nxt = {**_value(), **NEXT} + verdict = _judge([_beat("a"), _beat("b", value=nxt)]) + assert (verdict.refusals, verdict.booting, verdict.live, verdict.keys_fps) == ([], [], 2, {"k"}) + + +def test_a_live_process_with_loom_cell_unset_or_another_cell_refuses_at_once() -> None: + verdict = _judge([_beat("a", cell_name=""), _beat("b", cell_name="other")], "wait") + assert _reasons(verdict) == [ + "process a: runs with LOOM_CELL unset: set LOOM_CELL=first on it", + "process b: runs with LOOM_CELL=other: set LOOM_CELL=first on it", + ] + + +def test_a_stopped_unset_process_refuses_or_is_waited_on_as_asked() -> None: + stopped = [ + _beat("a", cell_name="", live=False, stopped=True), + _beat("b", cell_name=None, live=False, stopped=True), + ] + refused = _judge(stopped, "refuse") + assert [r.subject for r in refused.refusals] == ["process a", "process b"] + assert "may still be writing" in refused.refusals[0].reason + waited = _judge(stopped, "wait") + assert (waited.refusals, waited.aging) == ([], ["a", "b"]) + + +def test_a_live_instance_with_no_cell_row_is_booting() -> None: + verdict = _judge([_beat("a", cell_name=None)]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_cell_row_without_the_current_keys_refuses_even_while_booting() -> None: + old = {k: v for k, v in _value().items() if k not in ("serves", "marked", "router")} + verdict = _judge([_beat("a", value=old, mode="")]) + assert _reasons(verdict) == [ + "process a: reports no serves, marked or router key: upgrade it to this release" + ] + + +@pytest.mark.parametrize( + "keys,booting,refused", + [ + ({"marked": False}, [], "lose the tenancy marker"), + ({"marked": None}, ["a"], ""), + ({"router": False, "router_why": "a 404"}, [], "cannot read its router: a 404"), + ({"router": None}, ["a"], ""), + # A workers-only pod places no org: it needs no router credential. + ({"serves": False, "router": False}, [], ""), + ({"serves": False, "router": None}, [], ""), + ], +) +def test_marked_and_router_refuse_or_wait( + keys: dict[str, Any], booting: list[str], refused: str +) -> None: + verdict = _judge([_beat("a", value=_value(**keys))]) + assert verdict.booting == booting + assert (refused in " ".join(_reasons(verdict))) and bool(verdict.refusals) == bool(refused) + + +def test_no_tenancy_mode_yet_is_booting() -> None: + # A pod whose first cell-state read failed: mode '', marked and router null. + verdict = _judge([_beat("a", mode="", value=_value(marked=None, router=None))]) + assert (verdict.refusals, verdict.booting) == ([], ["a"]) + + +def test_a_fingerprint_on_neither_current_nor_next_refuses() -> None: + drifted = {**_value(), **cell.as_value({**FIELDS, "oidc_client_id": "drifted"})} + assert _reasons(_judge([_beat("a", value=drifted)])) == [ + "process a: runs neither loom.cell_self's current nor its next configuration" + ] + + +def test_the_final_refusals() -> None: + scim = {**_value(), **cell.as_value({**FIELDS, "scim": True})} + verdict = beats.judge( + [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" + ) + assert [r.reason for r in beats.final_refusals(verdict)] == [ + "SCIM is on: attach with SCIM on needs the per-cell SCIM release" + ] + two = _judge([_beat("a"), _beat("b", keys_fp="other")]) + assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason + assert "no live process" in beats.final_refusals(_judge([]))[0].reason + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock: + c = _Clock() + + async def _sleep(s: float) -> None: + c.now += s + + monkeypatch.setattr(apg, "_sleep", _sleep) + return c + + +@pytest.mark.asyncio +async def test_settle_waits_for_a_booting_process(clock: _Clock) -> None: + reads = [[_beat("a", cell_name=None)], [_beat("a", cell_name=None)], [_beat("a")]] + + async def read() -> list[beats.Beat]: + return reads.pop(0) + + wait = apg.Wait(10, clock) + verdict = await beats.settle(read, _judge, wait) + assert (verdict.live, wait.spent) == (1, 2.0) + + +@pytest.mark.asyncio +async def test_settle_refuses_at_the_end_of_wait_naming_what_it_waited_on(clock: _Clock) -> None: + async def read() -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name="", live=False, stopped=True)] + + with pytest.raises(Refused) as e: + await beats.settle(read, lambda f: _judge(f, "wait"), apg.Wait(3, clock)) + assert [str(r) for r in e.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: stopped with LOOM_CELL unset, not aged out within --wait", + ] + + +@pytest.mark.asyncio +async def test_settle_extends_once_for_a_process_first_seen_booting(clock: _Clock) -> None: + seen = 0 + + async def read() -> list[beats.Beat]: + nonlocal seen + seen += 1 + # A stopped process aging out first; then one booting from the third read (2 s + # spent), done at the seventh, past the first deadline. + if seen < 3: + return [_beat("a"), _beat("x", cell_name="", live=False, stopped=True)] + return [_beat("a"), _beat("b", cell_name=None)] if seen < 7 else [_beat("a"), _beat("b")] + + wait = apg.Wait(3, clock) + verdict = await beats.settle(read, lambda f: _judge(f, "wait"), wait, extend_s=4) + assert (verdict.live, wait.seconds) == (2, 6) + + async def forever() -> list[beats.Beat]: + return [_beat("b", cell_name=None)] + + wait = apg.Wait(2, clock) + with pytest.raises(Refused, match="still booting"): + await beats.settle(forever, _judge, wait, extend_s=1) + # Seen booting from the start: first sight plus the extension is inside the deadline. + assert wait.seconds == 2 + + +@pytest.mark.asyncio +async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: + wait = apg.Wait(2, clock) + clock.now += 100 # a scan's time + assert await wait.pause() and await wait.pause() + assert not await wait.pause() + assert wait.spent == 2.0 + + +# -- the rescan's watch ---------------------------------------------------------------- + + +def test_the_prune_contract_is_an_hour() -> None: + assert instances.PRUNE_AFTER_S == 3600 + + +@pytest.mark.asyncio +async def test_the_watch_refuses_every_unset_process_it_saw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rounds = [ + [ + _beat("a", cell_name=""), + _beat("c"), + _beat("d", cell_name=None, live=False, stopped=True), + ], + # a's rows were swept; c's cell row alone was swept: it was seen with the cell. + [_beat("c", cell_name=None, live=False, stopped=True)], + ] + + async def read_beats(_ch: Any, *, since: str | None = None) -> list[beats.Beat]: + assert since == "2026-01-01 00:00:00.000" + return rounds.pop(0) + + monkeypatch.setattr(beats, "read_beats", read_beats) + clock = _Clock() + watch = beats.Watch(None, "2026-01-01 00:00:00.000", "first", clock) + await watch.read_once() + await watch.read_once() + assert [r.subject for r in watch.refusals()] == ["process a", "process d"] + clock.now += beats.watch_period_s() + 1 + monkeypatch.setattr(beats, "read_beats", read_beats) + rounds.append([]) + await watch.read_once() + assert watch.refusals()[-1].subject == "heartbeats" + + +def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) + assert beats.Watch(None, "", "first").refusals_up_front() + monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) + assert beats.Watch(None, "", "first").refusals_up_front() == [] + + +# -- batches, ordering, arguments -------------------------------------------------------- + + +def test_batches_keep_the_count_and_the_byte_bound() -> None: + many = [f"org-{i:05d}" for i in range(ATTACH_BATCH + 1)] + assert [len(b) for b in cell_attach.batches(many, "dep")] == [ATTACH_BATCH, 1] + # Escaping grows a non-ASCII name six times: the encoded body is what counts. + wide = ["é" * 2000 + str(i) for i in range(10)] + out = cell_attach.batches(wide, "dep") + assert len(out) > 1 and sum(map(len, out)) == 10 + for b in out: + body = json.dumps({"deployment_id": "dep", "orgs": b}, separators=(",", ":")) + assert len(body.encode()) <= MAX_BODY_BYTES + + +def test_move_sources_are_scanned_first() -> None: + tables = [Table("loom", n, "MergeTree") for n in ("a", "memory_extraction_queue", "b", "src")] + ordered = cell_scan.order_sources(tables, {"loom.src"}) + assert [t.name for t in ordered] == ["memory_extraction_queue", "src", "a", "b"] + + +def _args(**over: Any) -> argparse.Namespace: + fields: dict[str, Any] = { + "cell": "first", + "wait": 10, + "scan_budget": 100, + "dry_run": False, + "ch_allow_user": [], + "ch_allow_user_directory": [], + "ch_allow_object": [], + "ch_allow_replica": [], + "pg_allow_role": [], + "router": "http://127.0.0.1:7900", + } + return argparse.Namespace(**{**fields, **over}) + + +def test_the_arguments(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") + assert cell_attach._args_problem(_args()) == "" + assert "LOOM_CELL" in cell_attach._args_problem(_args(cell="other")) + assert "--scan-budget" in cell_attach._args_problem(_args(scan_budget=0)) + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + assert "Postgres" in cell_attach._args_problem(_args()) + + +def test_an_org_past_the_limit_is_named_with_its_sources() -> None: + found = cell_scan.ScanResult() + found.add("postgres loom.tenancy", "x" * (cell.ORG_MAX_BYTES + 1)) + (refusal,) = cell_attach._too_long(found) + assert refusal.subject.startswith(f"org of {cell.ORG_MAX_BYTES + 1} bytes") + assert "found in postgres loom.tenancy" in refusal.reason + + +# -- the router ------------------------------------------------------------------------ + + +def _router(*responses: httpx.Response) -> cell_attach.Router: + answers = list(responses) + + def handler(_request: httpx.Request) -> httpx.Response: + return answers.pop(0) + + return cell_attach.Router( + httpx.URL("http://127.0.0.1:7900"), "first", httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_the_router_calls_refuse_as_the_router_answers() -> None: + ok = {"cell": "first", "state": "attaching"} + assert await _router(httpx.Response(200, json=ok)).call("") == ok + with pytest.raises(Refused, match="not upgraded"): + await _router(httpx.Response(404, json={"error": "no route"})).call("/attach/place", {}) + with pytest.raises(Refused, match=r"refused /attach/begin \(409\): use loom cell update"): + await _router(httpx.Response(409, json={"error": "use loom cell update"})).call( + "/attach/begin", {} + ) + with pytest.raises(Refused, match="did not answer as the router does"): + await _router(httpx.Response(502, text="bad gateway")).call("") + + +# -- the Postgres reads, on a fake session --------------------------------------------- + + +@dataclass +class FakeConn: + """Answers by the first needle the SQL holds; a callable answer is called with the + arguments; an exception answer is raised.""" + + routes: list[tuple[str, Any]] = field(default_factory=list) + executed: list[str] = field(default_factory=list) + + def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else answer + raise AssertionError(f"unrouted: {sql[:80]}") + + async def fetch(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchrow(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def fetchval(self, sql: str, *args: Any) -> Any: + return self._answer(sql, args) + + async def execute(self, sql: str, *args: Any) -> str: + self.executed.append(sql) + for needle, answer in self.routes: + if needle in sql: + if isinstance(answer, BaseException): + raise answer + return answer(*args) if callable(answer) else str(answer) + return "OK" + + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + + return _Tx() + + +def _seq(*values: Any) -> Callable[..., Any]: + pending = list(values) + return lambda *_a: pending.pop(0) + + +@pytest.mark.asyncio +async def test_the_session_checks_refuse_a_pooler() -> None: + moved = FakeConn( + [ + ("pg_backend_pid() AS pid", _seq((1, "t0"), (2, "t1"))), + ("current_setting($1, true)", None), + ] + ) + refusals = await apg.direct_refusals(moved, "attach") + assert [r.reason.split(":")[0] for r in refusals] == [ + "attach's session moved between backends", + "attach's session loses loom.tenancy_checked", + ] + direct = FakeConn([("pg_backend_pid() AS pid", (1, "t0")), ("current_setting($1, true)", "on")]) + assert await apg.direct_refusals(direct, "attach") == [] + assert "RESET ALL" in direct.executed + + +@pytest.mark.asyncio +async def test_the_lock_refuses_when_held_or_not_on_this_backend() -> None: + holder = [{"pid": 7, "application_name": "loom", "client": "10.0.0.1", "backend_start": "t"}] + held = FakeConn([("pg_try_advisory_lock", False), ("FROM pg_locks l LEFT JOIN", holder)]) + with pytest.raises(Refused, match="held by pid 7 \\(loom, from 10.0.0.1"): + await apg.take_lock(held, "attach") + pooled = FakeConn([("pg_try_advisory_lock", True), ("SELECT EXISTS", False)]) + with pytest.raises(Refused, match="a pooler"): + await apg.take_lock(pooled, "attach") + assert any("tcp_keepalives_idle" in s for s in pooled.executed) + + +@pytest.mark.asyncio +async def test_a_database_not_utf8_refuses() -> None: + assert await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "UTF8")])) == [] + (r,) = await apg.encoding_refusals(FakeConn([("pg_encoding_to_char", "SQL_ASCII")])) + assert "SQL_ASCII, not UTF8" in r.reason + + +def test_privilege_holders_outside_the_allowed_refuse() -> None: + holders = apg.Holders( + own=["loom"], + others={"ci": ["may insert into loom.tenancy"], "servers": ["owns loom.memory_current"]}, + ) + assert [r.subject for r in apg.privilege_refusals(holders, ["servers"])] == ["postgres role ci"] + assert apg.privilege_refusals(holders, ["ci", "servers"]) == [] + + +@pytest.mark.asyncio +async def test_settings_and_subscriptions_refuse() -> None: + conn = FakeConn( + [ + ( + "pg_db_role_setting", + [{"role": "ci", "db": "every database", "cfg": "Loom.Tenancy_Checked=on"}], + ), + ("pg_subscription", [{"subname": "mirror", "subenabled": False}]), + ] + ) + refusals = await apg.unmarked_refusals(conn, True) + assert [r.subject for r in refusals] == [ + "postgres unmarked session", + "postgres setting for ci in every database", + "postgres subscription mirror", + ] + + +@pytest.mark.asyncio +async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: + conn = FakeConn( + [ + ("SELECT current_user", "attacher"), + ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), + ("max_prepared_transactions", 5), + ] + ) + refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + assert [str(r) for r in refusals] == [ + "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " + "session, so step 4 could not wait for it; set it to 0", + ] + + +def _row(**over: Any) -> dict[str, Any]: + return { + "pid": 9, + "backend_type": "client backend", + "application_name": "ci", + "xact_start": "t", + "ours": True, + "xid_old": False, + "xmin_old": False, + "hidden": False, + **over, + } + + +@pytest.mark.asyncio +async def test_step_4_refuses_what_it_cannot_wait_for(clock: _Clock) -> None: + base = [("max_prepared_transactions", 0), ("pg_prepared_xacts", 0)] + hidden = FakeConn([*base, ("pg_stat_activity s", [_row(hidden=True)])]) + with pytest.raises(Refused, match="hidden from attach's role"): + await apg.wait_transactions(hidden, 100, apg.Wait(5, clock)) + prepared = FakeConn([("max_prepared_transactions", 0), ("pg_prepared_xacts", 1)]) + with pytest.raises(Refused, match="prepared transaction"): + await apg.wait_transactions(prepared, 100, apg.Wait(5, clock)) + held = FakeConn( + [ + *base, + ( + "pg_stat_activity s", + [_row(xmin_old=True), _row(pid=10, ours=False, xid_old=True), _row(pid=11)], + ), + ] + ) + with pytest.raises(Refused) as e: + await apg.wait_transactions(held, 100, apg.Wait(2, clock)) + assert [r.reason for r in e.value.refusals] == [ + "may have read `recording`: still open", + "another database's transaction, holding the snapshot horizon", + ] + done = FakeConn([*base, ("pg_stat_activity s", [_row(pid=10, ours=False, xid_old=True)])]) + await apg.wait_transactions(done, 100, apg.Wait(0, clock)) + + +class _Timeout(Exception): + sqlstate = "55P03" + + +@pytest.mark.asyncio +async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: + tries = _seq(_Timeout(), "UPDATE 1") + + def lock(*_a: Any) -> str: + answer = tries() + if isinstance(answer, Exception): + raise answer + return "LOCK TABLE" + + holders = _seq( + [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] + ) + conn = FakeConn( + [ + ("LOCK TABLE", lock), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", holders), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + confirmed: list[bool] = [] + + async def confirm() -> None: + confirmed.append(True) + + async def ready() -> bool: + return True + + logged: list[str] = [] + await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + assert logged == ["the mode's lock is busy; retrying"] + # Before each try and the one termination. + assert len(confirmed) == 3 + + +@pytest.mark.asyncio +async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: + conn = FakeConn( + [ + ( + "l.mode = 'ExclusiveLock'", + [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], + ), + ("rolsuper FROM pg_roles", False), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused, match="a superuser must end it"): + await apg.end_mode_transactions(conn, confirm) + + +@pytest.mark.asyncio +async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( + clock: _Clock, +) -> None: + conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + + async def confirm() -> None: + pass + + async def booting() -> bool: + return False + + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + assert not any("LOCK TABLE" in s for s in conn.executed) + + +@pytest.mark.asyncio +async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) + + async def read(mode: str, xid: int) -> apg.Durable: + return apg.Durable(mode=mode, first=(True, "first"), xid=xid) + + monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + with pytest.raises(Refused, match="did not land"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + with pytest.raises(Refused, match="is not above 100"): + await run.durable() + monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + assert await run.durable() == 100 + + async def waiting() -> str: + return "waiting" + + monkeypatch.setattr(pg_profile, "standby_now", waiting) + with pytest.raises(Refused, match="not synchronous before"): + await run.durable() + + +@pytest.mark.asyncio +async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + state = ["enforced"] + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return state[0], (True, "first") + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "read_mode", read_mode) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + assert await run.done() + state[0] = "recording" + with pytest.raises(Refused, match="run loom cell restore"): + await run.done() + + +def test_refusal_text() -> None: + assert str(Refusal("a", "b")) == "a: b" diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 3f670d4014..decb365521 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -695,7 +695,7 @@ def test_doctor_runs_the_cell_checks_only_with_loom_cell( monkeypatch.setattr(doctor, "_doctor_scim_token_check", lambda _s: ("scim_token", "PASS", "")) called: list[bool] = [] monkeypatch.setattr( - doctor_cell, "cell_checks", lambda: called.append(True) or [("cell_state", "PASS", "")] + doctor_cell, "cell_checks", lambda *_: called.append(True) or [("cell_state", "PASS", "")] ) names = [n for n, _g, _d in doctor._run_live_checks()] assert called == ([True] if runs else []) and ("cell_state" in names) == runs diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index e9ea657df6..c978eb94e2 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import contextlib import json from datetime import UTC, datetime from typing import Any @@ -29,6 +30,8 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). + monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) def _args(**over: Any) -> argparse.Namespace: diff --git a/tests/unit/test_doctor_client_checks.py b/tests/unit/test_doctor_client_checks.py index 554220a424..a2e1b0bd01 100644 --- a/tests/unit/test_doctor_client_checks.py +++ b/tests/unit/test_doctor_client_checks.py @@ -572,7 +572,7 @@ def test_doctor_client_flag_appends_the_lines_and_json_carries_them( ) -> None: from server.cli import doctor - monkeypatch.setattr(doctor, "_run_doctor_checks", lambda: [("python", "PASS", "3.12")]) + monkeypatch.setattr(doctor, "_run_doctor_checks", lambda *_: [("python", "PASS", "3.12")]) monkeypatch.setattr(doctor, "_doctor_problems", lambda: []) monkeypatch.setattr( doctor, diff --git a/tests/unit/test_doctor_client_client_only.py b/tests/unit/test_doctor_client_client_only.py index 60ceb323d6..712c10d7cd 100644 --- a/tests/unit/test_doctor_client_client_only.py +++ b/tests/unit/test_doctor_client_client_only.py @@ -56,7 +56,7 @@ def test_clickhouse_line_names_the_probed_host_on_failure(monkeypatch: pytest.Mo assert detail.startswith("https://ch.example.com:8443 → TimeoutError") -def _live() -> list[Check]: +def _live(*_roles: Any) -> list[Check]: return [ ("python_version", "PASS", "3.12"), ("clickhouse_reachable", "FAIL", "https://ch.example.com:8443 → TimeoutError: timed out"), From 1f059356679b1a5c20f36770a2bf70f1a702ab12 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:19:41 +0000 Subject: [PATCH 019/181] Attach's runbook, the cells.md first-attach steps and the trigger-cost gate measured end to end docs/operations/cell-attach.md is the operator's procedure: the preconditions attach checks and those it relies on, the flags, the point of no return at the mode commit, the orgs attach does not place and the manual removal of an org past the length limit, and doctor's reads after serving. cells.md describes the first attach as it runs (the lock, the refusals, the trigger pass, the mode commit and its durable read, step 4's wait, rescan and final checks), with its stated limits and failure rows. upgrades.md step 18 rotates the ClickHouse writer credential after the LOOM_CELL rollout. bench/cell_trigger_gate.py measures memory.set and a delivery-sized insert through an attached cell with the org triggers on and off; results.md records a local run. Attach compares the final root-secret fingerprint with the one its begin wrote, and the nonce cleanup's message names no single command. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 117 ++++++++++++ docs/architecture/cells.md | 290 +++++++++++++++++++---------- docs/benchmarks/results.md | 31 ++- docs/operations/cell-attach.md | 148 +++++++++++++++ docs/operations/postgres.md | 4 + docs/operations/upgrades.md | 8 + docs/reference/cli.md | 3 +- llms.txt | 1 + server/cli/cell_attach.py | 10 +- server/cli/cell_prepare.py | 2 +- tests/unit/snapshots/cli_help.json | 12 +- tests/unit/snapshots/cli_tree.json | 261 +++++++++++++++++++++++++- tests/unit/test_cell_attach.py | 17 ++ 13 files changed, 791 insertions(+), 113 deletions(-) create mode 100644 bench/cell_trigger_gate.py create mode 100644 docs/operations/cell-attach.md diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py new file mode 100644 index 0000000000..e90fbb32fa --- /dev/null +++ b/bench/cell_trigger_gate.py @@ -0,0 +1,117 @@ +"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server +of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an +unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` +guard stays). Run it against an attached cell's server, which must not serve anything else, +with a writer token for a placed org; it drops and recreates the cell's org triggers, so +never against a deployment in use. Needs no provider. + + PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ + --base-url http://127.0.0.1:7932 --pg-dsn \\ + --token --org --namespace +""" + +from __future__ import annotations + +import argparse +import asyncio +import statistics +import time +import uuid + +import asyncpg +import httpx + +from server.core import cell_triggers + +CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 + + +async def _triggers(dsn: str, on: bool) -> None: + """Create or drop the org triggers, and check that what was asked holds: every guarded + table carries both triggers enabled at origin, or none does.""" + conn = await asyncpg.connect(dsn) + try: + guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] + for table in guarded: + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + enabled = await conn.fetchval( + "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", + list(cell_triggers.ORG_TRIGGERS), + ) + finally: + await conn.close() + expected = 2 * len(guarded) if on else 0 + if enabled != expected: + raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + + +async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: + """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" + lat = [] + headers = {"Authorization": f"Bearer {args.token}"} + async with httpx.AsyncClient(timeout=30, headers=headers) as client: + for i in range(CALLS + WARMUP): + body = { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": f"gate-{uuid.uuid4().hex[:10]}", + "value": f"trigger gate {i}", + } + t = time.perf_counter() + (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() + if i >= WARMUP: + lat.append((time.perf_counter() - t) * 1000) + lat.sort() + return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] + + +async def _batch(args: argparse.Namespace, marked: bool) -> float: + """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" + settings = {"loom.tenancy_checked": "on"} if marked else {} + conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + try: + times = [] + for _ in range(INSERTS): + t = time.perf_counter() + await conn.execute( + "INSERT INTO loom.scope_generations (org, scope) " + "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + args.org, + uuid.uuid4().hex[:8], + ) + times.append((time.perf_counter() - t) * 1000) + return statistics.median(times) + finally: + await conn.close() + + +async def main() -> None: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--base-url", required=True) + p.add_argument("--pg-dsn", required=True) + p.add_argument("--token", required=True) + p.add_argument("--org", required=True) + p.add_argument("--namespace", required=True) + args = p.parse_args() + try: + for _ in range(ROUNDS): + for on in (True, False): + await _triggers(args.pg_dsn, on) + p50, p99 = await _memory_set(args) + marked, unmarked = await _batch(args, True), await _batch(args, False) + print( + f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " + f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " + f"{unmarked:.1f} ms unmarked", + flush=True, + ) + finally: + await _triggers(args.pg_dsn, True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..96580ec7d2 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -389,7 +389,8 @@ open meanwhile, as on a deployment never prepared, since the cell's Postgres, wh holds its state, cannot tell the two apart (ClickHouse carries no cell state: adding it would make the write path read a second store for tenancy) and such a process never reads the directory; where every process has `LOOM_CELL` unset, `loom doctor`'s naming -of it (which reads the directory, and ships with attach) is the only signal. A +of it (which reads the directory, and ships with the router's read-only scope) is the only +signal. A `(true, …)` row with `recording` and the own row: the first cell recording. Both present with the own row otherwise: `enforced`, a later cell's mode reading as `enforced` whatever its row says (its triggers, mode cache and CLI treat it so, and @@ -676,20 +677,18 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after any error in that -transaction it ends that session's backend (`pg_terminate_backend`, the backend named by -the pid and `backend_start` attach captured when it connected) and waits until it has -left `pg_stat_activity`, then reads the mode back before it retries or reports; a mode -read back as `enforced` is written again in a fresh transaction, whose commit waits for -the standby, before attach goes on, since a commit ended while it waited is on the -primary alone. The attach command fixes those timeouts, and its tests cover each of -these cases. The mode cache reads the mode with a plain `SELECT`, which takes no row -lock and never waits on attach. An insert that finds `enforced` (a process whose cache +server-side timeouts, then writes the mode and commits; after a failed try it ends the +mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in +this database (a pid captured at connect is wrong behind a connection path that swaps +backends), and waits until the lock clears. Whether the commit landed is decided by a +durable read of the mode, whose commit waits for the standby, since a commit ended while +it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never +waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in -`enforced`, and a CLI command refuses as in `enforced`, naming the org. A process whose -cache still reads `enforced` after step 3 or step 4 returns the cell to `recording` places -through the router's gate until its cache expires. -`loom doctor` lists any tenancy row the directory does not place. +`enforced`, and a CLI command refuses as in `enforced`, naming the org. A first attach +never lowers the mode once it commits `enforced`. `loom doctor`, with the router's +read-only scope, lists any tenancy row the directory does not place. - **It runs after verification, before anything is written for the org.** The op gate (`gates.authorize`) checks the org the transport resolves (`gates.resolve_org`, with @@ -983,51 +982,126 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. -The first attach turns the existing deployment into the first cell, in order: - -1. check the mode is `recording`, as prepare set it: every verified request has written - its org's tenancy row if absent while the directory answered that the cell is - `attaching`; -2. in one directory transaction, write `attach_started_at`, the cell's issuers, patterns - and fingerprint (both values of `loom.cell_self`), and the placements, then the - tenancy rows, of every org the cell holds, from one snapshot, straight into the - directory (no other cell exists yet): its configured orgs (of both values), the - distinct `org` of every table of both schemas (generated from the schema, as - erasure's table classification is), and the orgs of catalogued bucket objects; never - `''`, `'-'` or `'*'`; -3. set `enforced` as above, within `--wait`, which the wait below then shares, starting - no try that could commit the mode with less left than that wait's minimum (the mode - cache's lifetime plus the longest request deadline): attach then refuses, reporting - the mode it reads back (`recording`, and a rerun resumes from step 1), or the mode as - unknown when it cannot read it, and a rerun then resumes by the mode it reads; then - wait, within what remains of `--wait` (default 10 minutes, then refuse and set - `recording` back), until every live process reports `enforced` and then the mode - cache's lifetime plus the longest request deadline have passed; then run the SCIM - precondition again (`--revoke-unprovisioned` again if given), for accounts made - before `enforced` took hold; if it refuses, the cell stays `enforced` and - `attaching`, and running attach again resumes from the wait, as a resumed attach does - (below); -4. wait, within what remains of `--wait`, until no transaction that could have read - `recording` is still open, judged from a Postgres snapshot attach takes once it has - read `enforced` back in this run (not from a clock): a transaction on the cell's - database that started before it, from any client, background worker or logical - replication connection, read-only ones included, a writing transaction on another - database of the same server that did (it holds back every snapshot's horizon, and is - named as such), a prepared transaction (which refuses at once), or one whose state - attach cannot read counts as open, and Postgres's own maintenance processes, physical - replication connections (every standby's and a base backup's, which run no SQL) and - attach's own connections do not. Past `--wait` attach refuses, naming what it waited - on, and leaves the cell `enforced`, so a rerun resumes from step 3's wait. The attach - command fixes the exact query, and its tests cover each kind of transaction. Once the - wait passes, place the tenancy rows written since the snapshot and every org step 2's - sources yield when read again (the distinct `org` of every table of both schemas and - the orgs of catalogued bucket objects; a row or an object an unchecked writer, such - as an older CLI or a worker, wrote in `recording` after the snapshot is placed too), - check again that every live process reports `enforced` (refusing and returning to - `recording` as step 3 does if one does not), and mark the cell `serving`. The - directory records the highest mode attach has raised or tried to raise, which - `loom cell restore` writes back only upward: a cell whose step 3 was refused at the - lock is restored `enforced`, and its resumed attach then runs from step 3's wait. +The first attach (`loom cell attach`, its procedure on +[attaching the first cell](../operations/cell-attach.md)) turns the existing deployment +into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at +`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and +`loom cell update` take the same lock), after checking that the session is one direct +backend that keeps the tenancy marker: a lock left on a pooler's server connection would +refuse every later run, and attach's tenancy rows would be refused as an unchecked +writer's. A lock found lost between steps is taken again (refusing when another run holds +it), and the cell's state is read again. The lock does not make a run safe, since every +step is idempotent and the mode only rises; it keeps a second run's scans from holding the +snapshot horizon that step 4 waits on. + +It refuses, before any write, naming each: + +- stores other than the registered ones: the Postgres deployment id, ClickHouse's + `loom.storage_authority` and the shared host's `/v1/health/deployment` each name the + directory row's deployment id, `loom.cell_first` reads `(true, )`, the row is the + first cell's, `attaching`, with no other directory row; +- a database encoding other than UTF8 (prepare refuses the same), so a tenancy row + within the length limit is always an org a request can name; +- a synchronous standby not attached now, a durability profile that would not refuse + without one, or a role that may not write the durable read's logical message; a role + of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with + `max_prepared_transactions` above 0; +- **the tenancy-privilege check**: a login role, other than attach's own (superusers, + and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and + those named with `--pg-allow-role`, that holds or reaches through any membership a right + to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or + `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, + `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system + one (a function there can shadow one the servers call unqualified on their marked + session), `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` on + `loom.tenancy_checked`, membership in `pg_write_server_files` or + `pg_execute_server_program`, or ownership of a `loom` table, of the trigger functions or + of `loom.cell_first_immutable()`, the database or a schema; +- **the unmarked-session check**: a session that sends no marker reads one, or runs as + `replica`; a `pg_db_role_setting` entry of `loom.tenancy_checked` or + `session_replication_role` for this database or every one, matched case-insensitively; + any logical-replication subscription into this database, enabled or not (its apply + worker fires no origin trigger); +- the heartbeats (both tables, stopped rows included for the widened window, the live + window plus a stopping pod's drain and retirement): a live process with `LOOM_CELL` + unset or another cell, one without the heartbeat's keys, `marked` false, `router` false + where `serves` is true, a fingerprint on neither `loom.cell_self` value, several + `keys_fp` values, or none live; a booting process (no cell row yet, no mode, `marked` + or `router` null) is waited on within `--wait`; in `recording` a stopped process with + `LOOM_CELL` unset refuses too; +- the ClickHouse writer check and the scan's table list (`server/cli/cell_writers.py`, + `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user + reads. + +Then, in order: + +1. `attach/begin`, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, + through `attach/enforced` again, to the durable read. +3. Create the triggers itself, on every run, before step 2's scan and again just before the + mode commit (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads + every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an + `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with + the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only + org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such + orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row + is written; a name no request can carry is skipped and named. Then the tenancy rows of + the orgs found, on a marked session, and no placement: unscoped callers are answered + from the tenancy rows, and placing now would move the point of no return earlier. +5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of + `--scan-budget`, since step 4 reads the same tables again. Each refuses in + `recording`, which can still be backed out. +6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on + `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock + (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database + alone) and ended. From the commit on the mode only rises: every later refusal leaves + the cell `enforced` and `attaching`, and a rerun resumes from the durable read. +7. The durable read, in every run that reads `enforced`: the mode and the `first` row read + in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. + Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED + writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a + snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is + assigned after the mode commit. +8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends + on its cached mode (a request records a new org through `tenancy._RECORD`, which reads + the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling + one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A + prepared transaction, or a row of the cell's database whose state is hidden, refuses + at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the + heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); + the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery + tables move rows to ClickHouse), and the queue and drain tables and the tables a view + reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, + in `attach/place` batches; the writer check again; and the final checks, under a floor + of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): + the heartbeats, read through the rescan every half of the time the leader keeps a + stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` + unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` + equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 +hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would +be placed and skipped and the scan's time, and writes nothing; the router's checks at +`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is +done (exit 0), anything else refuses, naming `loom cell restore`. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1046,13 +1120,18 @@ cell with SCIM on admits no one until its first provisioning cycle has run. A ce nearing its stores' limits is closed to new orgs: its placement calls answer 503, so its issuers' new orgs wait for capacity rather than spill elsewhere. -A resumed first attach, for a cell whose directory row has `attach_started_at`, reruns -every refusal of the list above that applies to the first cell. It then continues by the -cell's mode: in `recording` from step 1; in `enforced`, from step 3's wait (every live -process reports `enforced`, then the cache's lifetime and the longest request deadline -pass) and then its SCIM precondition, so the SCIM check that ends an attach stopped -after setting `enforced`, whatever stopped it, always runs after `enforced` has taken -hold. +A resumed first attach reruns every refusal and continues by the directory row and the +cell's mode: in `recording` from the scan, in `enforced` from the durable read; whenever +the cell reads `enforced` it posts `attach/enforced` again, so a directory restored below +the cell's mode is raised. + +The checks of the stores' writers are point-in-time: a grant, a role setting, a +subscription, a ClickHouse writer or object added after `serving` passes them, as does a +configuration file attach cannot read (`postgresql.conf`, `postgresql.auto.conf`, a +standby's own) or a role that runs code as the server's OS user. `loom doctor` runs the +tenancy-privilege and unmarked-session reads again on every run and warns on what it finds +(`cell_tenancy_privilege`, `cell_unchecked_writers`); the runbook has the operator confirm +the rest. ## What each cell runs @@ -1102,9 +1181,9 @@ closed: every cell keeps serving the orgs it holds, routing continues from each replica's signed snapshot of its last cache (written to shared storage on every directory change), and the placements, lookups of uncached orgs, `loom cell` commands that read or write the directory and the recording cells' state reads answer 503 or -refuse until the directory is back, while `loom doctor` reports its directory checks as -not run and, with `LOOM_CELL` set, exits non-zero; rebuilding a lost directory from the -cells is a design of its own. +refuse until the directory is back, while `loom doctor`, with the router's read-only +scope, reports its directory checks as not run and, with `LOOM_CELL` set, exits +non-zero; rebuilding a lost directory from the cells is a design of its own. Each cell keeps, in its own Postgres, a copy of its directory row's configuration fingerprint (`loom.cell_self`), which its processes compare their configuration with and @@ -1200,30 +1279,36 @@ the directory row. ## Failure modes -| Failure | What happens | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | -| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | -| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | -| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | -| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | -| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | -| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | -| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | -| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | -| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | -| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | -| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | -| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | -| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | -| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | -| A recording insert held open during first attach | attach stops trying once too little of `--wait` is left for the wait, reporting the mode it reads back; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | -| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | -| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach | -| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| Failure | What happens | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| The directory is down | requests by host, by issuer and to the first cell are routed; new orgs and uncached lookups answer 503 | +| The router's placement call fails | the request answers 503 and nothing is written; orgs the cell holds are served | +| A cell's issuer places orgs in a flood | from `enforced` on, its placement calls answer 503 with `Retry-After` past its rate, and its calls for new orgs answer 503 past its quota if one is set; other cells place as before | +| A SCIM token of another cell, or an application at the wrong cell's host | the process does not boot, or `/scim/v2` answers 401 and writes nothing | +| A cell's Postgres or ClickHouse | that cell fails as a single deployment does ([storage authority](storage-authority.md#failure)); other cells serve | +| A cell's own rows or tenancy mode are unreadable | with `LOOM_CELL` set a process keeps its last rows and mode within the stale bound, then is gated, a server process whose first read fails does not boot (gated and unready with Postgres migrations pending), and a workers child that never read them is gated; with it unset a server process keeps its verdict with no bound, and a failed first read fails its boot (gated until a read lands with Postgres migrations pending) | +| A `first` or mode row removed, or both beside a tenancy or own row | the org triggers, where a cell has them, act as in `enforced` and every process is gated; `loom doctor` names it, a prepare rerun writes them back | +| No state, tenancy or own row on a cell the directory registers | open; `loom doctor` names it; `loom cell restore` closes it after a restore, a prepare rerun after a stopped prepare | +| A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | +| A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | +| A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | +| A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | +| The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | +| A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | +| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | +| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | +| A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | +| A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | +| A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | +| `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | +| A process that changes between attach's last check and the mode commit, or an HA manager that empties `synchronous_standby_names` between the checks around the durable read | not seen by attach: a window of seconds | +| A cell process with a longer `LOOM_INSTANCE_HEARTBEAT_SECONDS` than attach's | it can fall outside attach's live window while live; the runbook requires one value | ## Rolling upgrades @@ -1306,11 +1391,16 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). `loom doctor` names), with their functions in `schemas/pg_migrations/011_tenancy_triggers.sql` and `012_tenancy_guard.sql`; `server/cli/doctor_cell.py`: `loom doctor`'s cell checks; -- `server/cli/cell_scan.py`: the first attach's org scan, a library no command calls: the - tables of both stores with an `org` column, judged by engine and against erasure's - classification, every host of a cluster listing the same, and the distinct orgs of each - table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the - users, user directories, objects and replicas outside the operator's allow lists); +- `server/cli/cell_attach.py`: `loom cell attach`; `server/cli/cell_attach_pg.py`: its + Postgres reads and writes (the session checks, the attach lock, the tenancy-privilege + and unmarked-session checks doctor shares, the trigger pass, the mode commit, the + durable read, step 4's wait); `server/cli/cell_attach_beats.py`: its judgment of the + heartbeats and the rescan's watch; +- `server/cli/cell_scan.py`: the first attach's org scan: the tables of both stores with + an `org` column, judged by engine and against erasure's classification, every host of a + cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; + `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, + objects and replicas outside the operator's allow lists); `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; @@ -1386,14 +1476,14 @@ An integration test with two cells and a router in CI's Postgres job: CLI censuses cover the rest); - the first attach places every org with a row in any table, an org first seen during the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's snapshot and `enforced`, a ClickHouse row an older CLI wrote then and a + step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer request, passes its health probe, answers the probe route and runs no worker on an attached cell and on a later cell between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included after step 3 or step 4 returned the cell to it; a CLI command + `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token revocation) runs, also with Postgres down on a deployment never prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 07367d3688..81e099d3c8 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1220,8 +1220,35 @@ a row for the state read and the `loom.tenancy` lookup. Loom's own writes are ma this predicts the ship gate (a `memory.set` p99 rise of at most 10 ms, a delivery pass at most 10% slower) holds by a wide margin; the cost lands on scripts and older builds writing a cell without the check. This measures the triggers on the statements, not a -whole `memory.set` through a server on a cell: the gate itself is measured end to end -with attach. +whole `memory.set` through a server on a cell; the end-to-end gate follows. + +`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached +by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous +standby and ClickHouse in Docker on a shared host under other load), 400 sequential +`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert +of the held org from a marked and an unmarked session, the org triggers on and dropped in +turn, three rounds each; the harness checks the triggers it asked for are enabled: + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ + --pg-dsn --token --org --namespace +``` + +| Local, three rounds | Triggers on | Triggers off | +| ----------------------------------- | ---------------------- | ---------------------- | +| `memory.set` p50 | 154.4, 99.3, 98.4 ms | 91.9, 101.2, 101.6 ms | +| `memory.set` p99 | 494.0, 291.7, 257.6 ms | 279.2, 244.7, 233.3 ms | +| 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | +| 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | + +The marked batch, which is what a delivery pass of a cell's own processes writes, shows no +cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. +The `memory.set` figures move by more between rounds of one configuration than any trigger +cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms +here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no +consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the +host's noise, and the statement-level figures above bound the marked cost at under a +microsecond a row. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md new file mode 100644 index 0000000000..12a0cbe247 --- /dev/null +++ b/docs/operations/cell-attach.md @@ -0,0 +1,148 @@ +# Attaching the first cell + +`loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, +directory row `attaching`) into a `serving` one: it places every org the cell holds in the +router's directory, raises the cell's mode to `enforced` and creates its org triggers. This +page is the operator's procedure; the design is +[cells](../architecture/cells.md#attaching-a-cell). + +## Before attach + +Run attach at low load, as one run at a time, and not during a rollout, a provider +transition, a logical backup, a long erasure or a migration: each holds a transaction or a +snapshot long enough for step 4's wait to run out of `--wait`. Steps 2 and 4 run index skip +scans on the Postgres primary (each statement holds vacuum's horizon while it runs) and +read the `org` column of every ClickHouse table, which competes with serving load. + +Every item below is a precondition attach checks, or one it cannot check and relies on: + +1. Every router replica runs this release; every process and every CLI and CI image of + the cell runs it too ([upgrades](upgrades.md#upgrade-procedure), steps 16 and 17). +2. `LOOM_CELL` is set on every process of the cell. Every CLI and CI configuration that + writes for an org has `LOOM_CELL` set without `LOOM_CELL_ROUTER_TOKEN`: in `recording` + it then refuses only new orgs, so nothing has to change at the mode commit. With the + token removed, revoke the `INSERT` on `loom.tenancy` and the `UPDATE (since)` on + `loom.tenancy_mode` the CI role held for recording new orgs. +3. The ClickHouse writer credential is rotated after that rollout + ([upgrades](upgrades.md#upgrade-procedure), step 18). +4. No fingerprinted setting has changed since prepare, and none changes until attach ends: + a change gates every process against `loom.cell_self`. +5. `LOOM_SERVES_TRAFFIC=false` is set only on pods no load balancer routes to: attach + exempts them from the router check. +6. The cell's processes and the attach CLI share one `LOOM_INSTANCE_HEARTBEAT_SECONDS`: + attach judges which processes are live by its own value. +7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers + and for attach: attach refuses a session that moves between backends or loses the + tenancy marker, and refuses a server whose heartbeat reports `marked` false. +8. SCIM is off: attach refuses with it on. +9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the + cluster has an interserver `` or a `remote_servers` user equal to attach's. + One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an + availability cost, not a safety one. +10. Only Loom's own roles and superusers hold the rights attach's tenancy-privilege check + counts (inserting a tenancy row, changing the mode, creating in a schema, triggers on + `loom` tables, owning them or their trigger functions, `session_replication_role`, + `ALTER SYSTEM` of `loom.tenancy_checked`, `pg_write_server_files`, + `pg_execute_server_program`), and no role that holds one, or owns what it counts, owns + a `SECURITY DEFINER` function another role can execute that does any of that. Attach + checks the catalog; it cannot read a function's body for intent, or a role that runs + code as the server's OS user. +11. `postgresql.conf`, `postgresql.auto.conf` and any pending `ALTER SYSTEM` entry, on the + primary and on every standby, set neither `loom.tenancy_checked` nor + `session_replication_role`. Attach sees a live entry on the primary (its unmarked + session reads it), never a standby's files. +12. Postgres has a synchronous standby attached (`synchronous_standby_names` set) and + `max_prepared_transactions = 0`; the database encoding is UTF8. +13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. + +Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +`loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, +`loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on +`loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role +of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the +writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT +QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings +profile or quota capping the rows or bytes it reads, and the insert into +`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the +bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. + +## Run it + +```bash +loom cell attach --cell --dry-run \ + --ch-allow-user --pg-allow-role +loom cell attach --cell \ + --ch-allow-user --pg-allow-role +``` + +`--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs +per source table, the ones it would skip, and whether the scan fit in half of +`--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object +and replica that may write the configured databases with `--ch-allow-user`, +`--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every +Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each +list is audited by the router. `--wait` (default 600 s) bounds the waits together; +`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of +it, since step 4 reads the same tables again. + +A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a +rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it +is done. + +## The point of no return + +Step 3's mode commit is one-way. Before it, a refused attach leaves the cell `recording`, +its tenancy rows and its triggers in place (they pass every row in `recording`), and the +cell can be backed out (the first-cell branch of +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated)): +`loom cell remove` serves only a first cell that never left `recording`. After it, scoped +principals place their orgs live within seconds, every refusal leaves the cell `enforced` +and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unsetting +`LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` +always goes forward. + +From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account +routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an +ordinary mint scoped to its org places the org on its first request. A direct call of the +router's operator place route, or a live placement with the cell's credential, cannot read +the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's +credential is the servers' alone. + +## Orgs attach does not place + +- **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is + written. One first found in step 4 is skipped and named. Remove such an org by hand + until the stray-org erase command exists, in this order: its bucket objects, from the + keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no + memory content is left outside erasure; then its rows in every table with an `org` column + of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables + included; then its `loom.tenancy` row, on a session with `loom.tenancy_checked` set. +- **A name no request can carry** (not UTF-8, a NUL) found in a data table is skipped and + named; its rows stay outside placement. +- **An org found only in a log table** (`agent_events` and the tables derived from it, the + code search trace) is never placed: a log row keeps the org an unauthenticated caller + typed. Once that org is placed in another cell, its log rows here are outside its + erasure there. + +`loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, +smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left +unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available +yet. + +## After attach + +`loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres +role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an +org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or +a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a +writer added after `serving` passes attach's checks, which ran once; doctor is how it is +found. + +## Related + +- [Cells](../architecture/cells.md) +- [Troubleshooting: cells](troubleshooting.md#cells) +- [Upgrades](upgrades.md) diff --git a/docs/operations/postgres.md b/docs/operations/postgres.md index d087a158c4..895086c474 100644 --- a/docs/operations/postgres.md +++ b/docs/operations/postgres.md @@ -44,6 +44,10 @@ pooler refuses or drops (its org triggers then refuse Loom's own writes for an o cell holds no row for, and the `loom.tenancy` guard its tenancy row for each newly placed org), and `loom doctor` names a session that lost it. Scripts and bulk loaders never set `session_replication_role = replica` on a cell: it skips the org triggers and the guard. +No role is granted `SET` or `ALTER SYSTEM` on `session_replication_role`, `ALTER SYSTEM` +on `loom.tenancy_checked`, or membership in `pg_write_server_files` or +`pg_execute_server_program` on a cell: `loom cell attach` refuses such a role, and +`loom doctor` warns on one ([attaching the first cell](cell-attach.md)). Every replica of the deployment, and its worker process, sets the same values. The full list, with the admission and delivery settings, is in the diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8caf8c789d..50dd5fc46a 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -320,6 +320,14 @@ checkout. answers them from one read per replica per `LOOM_ROUTER_DIRECTORY_CACHE_S` ([cells](../architecture/cells.md#the-heartbeats-keys)). +18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and + CI configuration of the cell (step 16), rotate the ClickHouse credential they write + with: a new user or password for the cell's servers and CLIs, the old one dropped. No + CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` + unset; the rotation cuts off any the rollout missed before it can write an org's rows + past attach's last scan. Name the new user with `--ch-allow-user` + ([attaching the first cell](./cell-attach.md)). + While a migration applies, the old build keeps serving: every migration is additive. The rules that let the two builds serve side by side are in [rolling upgrades](../architecture/rolling-upgrades.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..12bac9a65b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -315,7 +315,7 @@ python scripts/codegen/docs_reference.py | `loom bot create` | Register a bot: its namespaces, and a key (--generate-key or --public-key) and/or a CI trust (--issuer, --audience, --claim). | `--name`, `--org`, `--namespace-role`, `--generate-key`, `--public-key`, `--issuer`, `--audience`, `--claim`, `--jwks-uri`, `--max-token-age`, `--max-access-token-ttl`, `--max-agents`, `--agent-claim`, `--platform-admin`, `--replace` | | `loom bot list` | Every bot, active or disabled. | | | `loom bot disable` | Refuse its proofs and revoke the tokens it holds. | `--name` | -| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--token` | +| `loom doctor` | Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. | `--json`, `--client`, `--server-url`, `--pg-allow-role`, `--stray-orgs`, `--stray-orgs-budget`, `--token` | | `loom report` | Print the bug form's Environment field: versions and the `loom doctor` checks. | `--open` | | `loom drops` | Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. | `--namespace`, `--org`, `--hours` | | `loom namespaces-gc` | List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per… | `--org`, `--prefix`, `--namespace`, `--exclude-prefix`, `--idle-hours`, `--include-registered`, `--erase`, `--no-purge`, `--skip-external`, `--batch`, `--show`, `--requester`, `--worker-pass` | @@ -330,6 +330,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index ca465459d3..43b8421b0e 100644 --- a/llms.txt +++ b/llms.txt @@ -104,6 +104,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. +- [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 368f74ebf8..c8b72fb687 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -1,6 +1,6 @@ """`loom cell attach`: turn a prepared first cell into a `serving` one (docs/architecture/cells.md, "Attaching a cell"; the runbook is -docs/operations/cells.md). +docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: @@ -121,7 +121,7 @@ def _too_long(found: cell_scan.ScanResult) -> list[Refusal]: Refusal( f"org of {len(org.encode())} bytes ({org[:64]}…)", f"found in {', '.join(sorted(sources))}: longer than {cell.ORG_MAX_BYTES} bytes; " - "remove it first (docs/operations/cells.md, the over-long org procedure)", + "remove it first (docs/operations/cell-attach.md, the over-long org procedure)", ) for org, sources in sorted(found.too_long.items()) ] @@ -174,7 +174,8 @@ def __init__( self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) - self.keys_fp = "" + self.keys_fp = "" # the live processes' one root-secret fingerprint, last read + self.begun_keys_fp = "" # the one this run's `attach/begin` wrote self.report: dict[str, Any] = {"cell": self.name} # -- the lock session -------------------------------------------------------------- @@ -410,6 +411,7 @@ async def begin(self) -> None: try: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -523,7 +525,7 @@ async def _read() -> list[beats.Beat]: verdict = await beats.settle( _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() ) - if verdict.keys_fps != {self.keys_fp}: + if verdict.keys_fps != {self.begun_keys_fp}: raise _refused( "processes", "the root-secret fingerprint changed during attach (a root-secret rotation): " diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index cf83fa8f27..4a820f652b 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -286,7 +286,7 @@ async def _bucket() -> None: try: await step() except Exception as e: # noqa: BLE001 — every nonce is tried; a stale one holds no data - print(f"loom cell prepare: the probe nonce in {where} stays: {e}", file=sys.stderr) + print(f"loom cell: the probe nonce in {where} stays: {e}", file=sys.stderr) def _register_timeout() -> httpx.Timeout: diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..3e88dbafca 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -397,8 +397,8 @@ "options: -h, --help show this help message and exit --name NAME" ], "loom doctor": [ - "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--token TOKEN]", - "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --token TOKEN Override the client's token." + "usage: loom doctor [-h] [--json] [--client] [--server-url SERVER_URL] [--pg-allow-role PG_ALLOW_ROLE] [--stray-orgs] [--stray-orgs-budget STRAY_ORGS_BUDGET] [--token TOKEN]", + "options: -h, --help show this help message and exit --json Emit the checks and the problem list as JSON. --client Also examine this machine as a coding-agent client: the server the hooks would call, the token they would present, whether it is accepted and can write the intended namespace, the Claude Code / Codex wiring, and capture jobs that never reached the server. --server-url SERVER_URL Override the client's server URL. --pg-allow-role PG_ALLOW_ROLE On a cell: a Postgres role that may make an org held (the servers' role). --stray-orgs On a cell: also read the orgs its stores hold without a tenancy row. --stray-orgs-budget STRAY_ORGS_BUDGET Seconds --stray-orgs may read for, smallest tables first (default 60). --token TOKEN Override the client's token." ], "loom report": [ "usage: loom report [-h] [--open]", @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {credential,limit,which,remove,prepare,attach} ...", + "positional arguments: {credential,limit,which,remove,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -459,6 +459,10 @@ "usage: loom cell prepare [-h] --cell CELL --host HOST --credential-sha256 SHA256 [--probe-url PROBE_URL] [--wait WAIT] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell attach": [ + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..3d597c1e03 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -11213,6 +11213,9 @@ "json", "client", "server_url", + "pg_allow_role", + "stray_orgs", + "stray_orgs_budget", "token" ] } @@ -11280,6 +11283,51 @@ "help": "Override the client's server URL.", "metavar": null }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "On a cell: a Postgres role that may make an org held (the servers' role).", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs" + ], + "dest": "stray_orgs", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "On a cell: also read the orgs its stores hold without a tenancy row.", + "metavar": null + }, + { + "option_strings": [ + "--stray-orgs-budget" + ], + "dest": "stray_orgs_budget", + "action": "_StoreAction", + "default": 60.0, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds --stray-orgs may read for, smallest tables first (default 60).", + "metavar": null + }, { "option_strings": [ "--token" @@ -12903,6 +12951,211 @@ "metavar": null } ] + }, + "attach": { + "prog": "loom cell attach", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_attach:cmd_attach" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "wait", + "scan_budget", + "dry_run", + "ch_allow_user", + "ch_allow_user_directory", + "ch_allow_object", + "ch_allow_replica", + "pg_allow_role", + "router" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds the waits share", + "metavar": null + }, + { + "option_strings": [ + "--scan-budget" + ], + "dest": "scan_budget", + "action": "_StoreAction", + "default": 7200, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:int", + "required": false, + "help": "Seconds one full org scan may take", + "metavar": null + }, + { + "option_strings": [ + "--dry-run" + ], + "dest": "dry_run", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run)", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user" + ], + "dest": "ch_allow_user", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user that may write the configured databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-user-directory" + ], + "dest": "ch_allow_user_directory", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse user directory the check cannot read", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-object" + ], + "dest": "ch_allow_object", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A ClickHouse object (db.name) that writes the databases", + "metavar": null + }, + { + "option_strings": [ + "--ch-allow-replica" + ], + "dest": "ch_allow_replica", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A replica outside the cluster (db.table:replica)", + "metavar": null + }, + { + "option_strings": [ + "--pg-allow-role" + ], + "dest": "pg_allow_role", + "action": "_AppendAction", + "default": [], + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "A Postgres role that may make an org held (the servers' role)", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + } + ] } }, "aliases_of": { @@ -12910,7 +13163,8 @@ "limit": "loom cell limit", "which": "loom cell which", "remove": "loom cell remove", - "prepare": "loom cell prepare" + "prepare": "loom cell prepare", + "attach": "loom cell attach" }, "choices_help": [ { @@ -12937,6 +13191,11 @@ "dest": "prepare", "help": "Register this cell in the router's directory and write its rows.", "metavar": "prepare" + }, + { + "dest": "attach", + "help": "Attach the prepared first cell: place its orgs, serve.", + "metavar": "attach" } ] } diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 015cebd948..1c725205e6 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -680,3 +680,20 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: def test_refusal_text() -> None: assert str(Refusal("a", "b")) == "a: b" + + +@pytest.mark.asyncio +async def test_a_root_secret_change_during_attach_refuses_at_the_end( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.begun_keys_fp = "before" + + async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: + return beats.Verdict(live=1, keys_fps={"after"}) + + monkeypatch.setattr(beats, "settle", settle) + with pytest.raises(Refused, match="a root-secret rotation"): + await run.final_checks(beats.Watch(None, "", "first")) + run.begun_keys_fp = "after" + await run.final_checks(beats.Watch(None, "", "first")) From 55c0918ae463b59d20c3aee84c3e74afd891e1fc Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:35:26 +0000 Subject: [PATCH 020/181] The widened heartbeat window lives with attach, so server.core imports no boot or insights module instances.read_recent takes the window it reads back; attach's heartbeat module computes it from the live window and a stopping process's drain and retirement budgets. PRUNE_AFTER_S is marked as the contract it is. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach_beats.py | 14 ++++++++++++-- server/core/instances.py | 25 ++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a2fc37701d..d1d938a46b 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -32,6 +32,16 @@ StoppedUnset = Literal["refuse", "wait"] +def widened_window_s() -> float: + """The live window plus what a stopping process still does after its stop beat: it + drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, + stopped or not, is younger than this may still write.""" + from server.boot import SHUTDOWN_DRAIN_BUDGET_S + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + return instances.live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S + + @dataclass(frozen=True) class Beat: """One process: its instance row (`live`: inside the live window and not stopped) and @@ -48,7 +58,7 @@ class Beat: async def read_beats(ch: Any, *, since: str | None = None) -> list[Beat]: - rows = await instances.read_recent(ch, since=since) + rows = await instances.read_recent(ch, window_s=widened_window_s(), since=since) ids = [str(r["instance_id"]) for r in rows] cells = await ch.query_all( f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " @@ -236,7 +246,7 @@ async def settle( def watch_period_s() -> float: """`T'`: how long a row whose window reaches past the rescan's start is kept before the leader's sweep may drop it, less a margin for clock skew.""" - return instances.PRUNE_AFTER_S - instances.widened_window_s() - _SKEW_S + return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S class Watch: diff --git a/server/core/instances.py b/server/core/instances.py index bb47bfd945..80e3e76896 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -56,7 +56,7 @@ # How old a row's last beat is before the leader drops its key. A contract across builds, # not a tunable: attach reads stopped rows back that far (`read_recent`) while the leader # that sweeps them may run another build, so a change is a design change of its own. -PRUNE_AFTER_S = 3600 +PRUNE_AFTER_S = 3600 # not a setting: a contract across builds # (pid, instance id, start time) of this process. _process: tuple[int, str, datetime] | None = None @@ -134,16 +134,6 @@ def live_window_s() -> float: return _LIVE_INTERVALS * float(settings.instance_heartbeat_seconds) -def widened_window_s() -> float: - """The live window plus what a stopping process still does after its stop beat: it - drains its ClickHouse writes, then settles cut-off ones. A process whose last beat, - stopped or not, is younger than this may still write.""" - from server.boot import SHUTDOWN_DRAIN_BUDGET_S - from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S - - return live_window_s() + SHUTDOWN_DRAIN_BUDGET_S + SHUTDOWN_RETIRE_S - - def boot_allowance_s() -> float: """How long a booting process takes to report its first cell row and its first router answer: two beats and one router read.""" @@ -217,10 +207,12 @@ async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, return out -async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, Any]]: - """Every process of every deployment sharing this ClickHouse whose widened window - (`widened_window_s`) reaches past `since` (a ClickHouse `DateTime64(3)` as text), or - past now without one: stopped rows included, which `read_live` drops. `stopped` is its +async def read_recent( + ch: Any, *, window_s: float, since: str | None = None +) -> list[dict[str, Any]]: + """Every process of every deployment sharing this ClickHouse whose last beat plus + `window_s` reaches past `since` (a ClickHouse `DateTime64(3)` as text), or past now + without one: stopped rows included, which `read_live` drops. `stopped` is its newest row's flag and `live` whether that row is inside the live window and not stopped, both judged in ClickHouse.""" rows = await ch.query_all( @@ -238,7 +230,7 @@ async def read_recent(ch: Any, *, since: str | None = None) -> list[dict[str, An """, params={ "live": int(live_window_s() * 1000), - "wide": int(widened_window_s() * 1000), + "wide": int(window_s * 1000), "since": since or "", }, ) @@ -397,5 +389,4 @@ def _reset_for_tests() -> None: "start", "stop", "tick", - "widened_window_s", ] From 4a7bc8804793d7b6832af214f89209adcf3f7c71 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 05:51:59 +0000 Subject: [PATCH 021/181] ensure_table's integration test lands with attach, its caller Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- .../integration/test_cell_org_triggers_pg.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index e49dde4426..a47f4baec5 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -714,6 +714,32 @@ async def test_missing_names_a_guard_disabled_narrowed_or_calling_another_functi assert "loom.tenancy_refuse_unmarked() has a body or settings no migration writes" in texts +@pytest.mark.asyncio +async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_asked() -> None: + """Attach creates a first cell's triggers in `recording`, one table at a time; a boot's + `ensure` never does, and the function passes every row there.""" + async with _cell("recording") as (dsn, conn): + with pytest.raises(RuntimeError, match="recording"): + await cell_triggers.ensure_table(conn, "scope_generations") + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + assert await _cell_triggers(conn) == { + ("scope_generations", "tenancy_insert", "O"), + ("scope_generations", "tenancy_update", "O"), + ("tenancy", "tenancy_record", "O"), + } + # Again: nothing to do. + assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + with pytest.raises(ValueError, match="no cell triggers"): + await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) + plain = await _unmarked(dsn) + try: + await plain.execute(_INSERT, "unheld-org", "s") # recording passes every row + await plain.execute(_HOLD, "new-org") # a first cell still recording records + finally: + await plain.close() + + @pytest.mark.asyncio @pytest.mark.parametrize("loom_cell", ["eu-1", ""]) async def test_the_cell_refresh_reads_the_marker_its_pool_sends( From 4193ffd36fcece18fa4bbb77ceef6f2fbd62ac78 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 07:48:23 +0000 Subject: [PATCH 022/181] Fix a double space in the failure table's pooler row Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 96580ec7d2..46e1439538 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1292,7 +1292,7 @@ the directory row. | A cell's `first` value is not yet read | every request but health probes and the probe route answers 503 (SCIM with `Retry-After`); the process is unready and runs no worker (a workers child, or a server process with Postgres migrations pending; any other server process fails its boot) | | A request for another cell's org | 409 `WRONG_CELL` after verification, nothing written | | A platform, `*`-org or principal-less call for an unheld org | on a first cell, 503 while the directory answers `attaching` or cannot be read before a `serving` answer; 409 `WRONG_CELL` once `serving` and on a later cell | -| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | +| A pooler that strips the tenancy marker from a cell's sessions | the `cell_marker` readiness check fails in `enforced` and `partial` and warns in `recording`; the heartbeat reports `marked` false and attach refuses the process | | A cell restored to an earlier time | `loom cell restore` writes placed orgs' rows back before it starts | | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | From c6501191d838adfbfee5585f65872f3af659332d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:10:11 +0000 Subject: [PATCH 023/181] Attach refuses a table its role cannot alter up front, charges --wait in wall time, and says when the mode commit's outcome is unknown The refusals name every table the trigger pass would alter whose owner role attach's role is not a member of, before any write; ensure_table and _run_ddl answer applied, busy or denied, so a denial during the pass refuses at once rather than after --wait as a busy lock. --wait is wall time: each bounded wait charges its elapsed time, lock tries included, and every lock try waits at most what is left; scan time stays outside it. From the mode commit on, a stop that comes before the durable read says the commit's outcome is unknown and to rerun attach, for a refusal and for any other error alike. The prune-window check runs with the refusals, before any write; an enforced resume runs one trigger pass; the lock session opens inside the block that closes it; the durable read retries a timeout or an outage only; refusals name no command that does not exist. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 78 +++--- server/cli/cell_attach_beats.py | 37 ++- server/cli/cell_attach_pg.py | 235 +++++++++++------- server/core/cell_triggers.py | 41 +-- tests/integration/test_cell_attach.py | 35 +++ .../integration/test_cell_org_triggers_pg.py | 7 +- tests/unit/test_cell_attach.py | 124 ++++++++- 7 files changed, 396 insertions(+), 161 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index c8b72fb687..ea9cf77d35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -365,9 +365,9 @@ async def refusals(self) -> tuple[Any, Any, cell_scan.TableList]: out += await self.catalog_refusals() out += await self.durability_refusals() rows = await self.lock_conn.fetch(cell_triggers.ORG_TABLES) - tables_pg = [f"loom.{r['relname']}" for r in rows] - out += await apg.own_role_refusals(self.lock_conn, tables_pg) + out += await apg.own_role_refusals(self.lock_conn, list(rows)) out += await self.ch_limit_refusals() + out += beats.prune_window_refusals() if out: raise Refused(out) state, _ = await apg.read_mode(self.lock_conn) @@ -449,6 +449,8 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg + # From here on the mode may have risen: only the durable read can say. + self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: @@ -465,22 +467,29 @@ async def commit_mode(self) -> None: async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg_profile + from server.core import pg, pg_profile - while True: - try: + with self.wait.counting(): + while True: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") - read = await apg.durable_read() + try: + read = await apg.durable_read() + except Exception as e: + # A try cut off by its bound, or an outage: its outcome is unknown, and + # the next try reads again. Anything else is no reason to retry. + outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) + if not (isinstance(e, TimeoutError) or outage): + raise + self.log(f"the durable read failed ({type(e).__name__}); retrying") + if not await self.wait.pause(): + raise _refused( + "postgres durable read", "did not commit within --wait" + ) from e + continue if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break - except Refused: - raise - except Exception as e: - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused("postgres durable read", "did not commit within --wait") from e if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -540,8 +549,6 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.cheap_checks("wait") since = str((await self.ch.query_one("SELECT toString(now64(3)) AS t"))["t"]) watch = beats.Watch(self.ch, since, self.name) - if refusals := watch.refusals_up_front(): - raise Refused(refusals) task = asyncio.create_task(watch.run()) try: listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) @@ -567,7 +574,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): raise _refused( "postgres loom.cell_self", - "differs from the directory's fingerprints (a loom cell update between them?)", + "differs from the directory's fingerprints (a fingerprinted setting changed " + "during attach?)", ) state, _ = await apg.read_mode(self.lock_conn) if state != "enforced": @@ -596,7 +604,9 @@ async def done(self) -> bool: if state != "enforced": raise _refused( "postgres cell state", - f"the directory says serving and the cell reads {state}: run loom cell restore", + f"the directory says serving and the cell reads {state}: the directory and the " + "cell's Postgres disagree, as after a restore of one of them; attach lowers " + "and repairs neither", ) return True @@ -606,8 +616,8 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - await self.open_lock() try: + await self.open_lock() await self.read_own_row() topology, names, listed = await self.refusals() if self.args.dry_run: @@ -626,7 +636,6 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.triggers() await self.router.call( "/attach/enforced", {"deployment_id": self.row["deployment_id"]} ) @@ -695,18 +704,30 @@ def log(text: str) -> None: except Refused as e: for r in e.refusals: log(f"refusing: {r}") - mode = ( - "enforced, attaching: rerun attach to resume" - if run.mode == "enforced" - else ("unchanged" if not run.mode else f"{run.mode}") - ) - log(f"the cell stays {mode}") + log(mode_line(run.mode)) + return 1 + except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes + log(f"stopped ({type(e).__name__}: {e}); run it again") + log(mode_line(run.mode)) return 1 run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code +def mode_line(mode: str) -> str: + """What a stopped run left the cell's mode at, by what it last knew.""" + if mode == "enforced": + return "the cell stays enforced and attaching: rerun attach to resume" + if mode == "unknown": + return ( + "the mode commit's outcome is unknown: rerun attach, which resumes by the mode it reads" + ) + if mode == "recording": + return "the cell stays recording" + return "the cell's state is unchanged" + + def cmd_attach(args: argparse.Namespace) -> int: if problem := _args_problem(args): print(f"loom cell attach: {problem}", file=sys.stderr) @@ -719,14 +740,7 @@ def cmd_attach(args: argparse.Namespace) -> int: from server.core.ch import statement_timeout async def _run(ch: Any) -> int: - try: - return await attach(ch, args, url) - except Exception as e: # noqa: BLE001 — a store that fails names itself; a rerun resumes - print( - f"loom cell attach: stopped ({type(e).__name__}: {e}); run it again", - file=sys.stderr, - ) - return 1 + return await attach(ch, args, url) # Every ClickHouse statement, the connect included, may run as long as a scan may. with statement_timeout(args.scan_budget): diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index d1d938a46b..a9c6e80377 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -209,6 +209,16 @@ async def settle( """Read and judge until nothing is waited on: Refused at the first refusal, and once `wait` runs out naming what was waited on. With `extend_s`, the deadline is extended once, to `extend_s` past the first sight of the last process first seen booting.""" + with wait.counting(): + return await _settle(read, judge_beats, wait, extend_s) + + +async def _settle( + read: Callable[[], Awaitable[list[Beat]]], + judge_beats: Callable[[list[Beat]], Verdict], + wait: Wait, + extend_s: float, +) -> Verdict: first_seen: dict[str, float] = {} extended = False while True: @@ -220,7 +230,7 @@ async def settle( raise Refused(refusals) return verdict for i in verdict.booting: - first_seen.setdefault(i, wait.spent) + first_seen.setdefault(i, wait.used()) if await wait.pause(): continue if extend_s and not extended and verdict.booting: @@ -249,6 +259,20 @@ def watch_period_s() -> float: return instances.PRUNE_AFTER_S - widened_window_s() - _SKEW_S +def prune_window_refusals() -> list[Refusal]: + """A heartbeat interval so long that the leader may sweep a stopped process's rows + before step 4's watch reads them refuses before any write.""" + if watch_period_s() < 2 * _READ_TIMEOUT_S: + return [ + Refusal( + "LOOM_INSTANCE_HEARTBEAT_SECONDS", + "so long that a stopped process's rows can be swept before attach reads " + "them: lower it", + ) + ] + return [] + + class Watch: """The processes seen during step 4's rescan, read every `T' / 2` from the rescan's start (`since`), so a process with `LOOM_CELL` unset that ran during it is refused at @@ -265,17 +289,6 @@ def __init__( self.gap = False self._good = clock() - def refusals_up_front(self) -> list[Refusal]: - if watch_period_s() < 2 * _READ_TIMEOUT_S: - return [ - Refusal( - "LOOM_INSTANCE_HEARTBEAT_SECONDS", - "so long that a stopped process's rows can be swept before attach reads " - "them: lower it", - ) - ] - return [] - async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 6ee4432907..076dc4a196 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -10,8 +10,9 @@ from __future__ import annotations import asyncio +import contextlib import time -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -23,38 +24,59 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval -# The migration lock's wait a trigger try (a boot holds it for its migrations). -_MIGRATION_LOCK_TIMEOUT = "10s" -# The mode commit's lock wait a try, and the statement bound above it. -_MODE_LOCK_TIMEOUT = "2s" -_MODE_STATEMENT_TIMEOUT = "5s" +# The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a +# try never waits longer than --wait has left. +_MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# The mode commit's lock wait a try, in ms, and its statement bound's margin above it. +_MODE_LOCK_MS = 2_000 # not a setting: one try's bound +_MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} class Wait: - """`--wait`: the seconds attach's waits share, counting only the time spent waiting - (never scan time).""" + """`--wait`: the wall-clock seconds attach's bounded waits share. Each wait runs inside + `counting()`, which charges its elapsed time, lock tries included; time outside every + wait (a scan) is never charged.""" def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: self.seconds = float(seconds) self.spent = 0.0 self._clock = clock + self._since: float | None = None + + def used(self) -> float: + """The seconds charged so far, the wait running now included.""" + running = 0.0 if self._since is None else self._clock() - self._since + return self.spent + running def left(self) -> float: - return max(0.0, self.seconds - self.spent) + return max(0.0, self.seconds - self.used()) + + @contextlib.contextmanager + def counting(self) -> Iterator[None]: + """Charge the block's wall time (a block inside another is charged once).""" + if self._since is not None: + yield + return + self._since = self._clock() + try: + yield + finally: + self.spent += self._clock() - self._since + self._since = None - def now(self) -> float: - return self._clock() + def bound_ms(self, most_ms: int) -> int: + """A lock timeout for one try: `most_ms`, or what is left when that is less.""" + return min(most_ms, int(self.left() * 1000)) async def pause(self, seconds: float = POLL_S) -> bool: - """Wait `seconds` (at most what is left) and count them; False once none is left.""" + """Wait `seconds` (at most what is left); False once none is left.""" step = min(seconds, self.left()) if step <= 0: return False - start = self._clock() - await _sleep(step) - self.spent += max(step, self._clock() - start) + with self.counting(): + await _sleep(step) return True @@ -137,7 +159,7 @@ async def take_lock(conn: Any, what: str) -> None: [ Refusal( "attach lock", - f"held by {holder}: an attach, prepare or update is running; {what} " + f"held by {holder}: an attach or a prepare is running; {what} " "runs one at a time", ) ] @@ -364,14 +386,22 @@ async def session_passes() -> bool: """ -async def own_role_refusals(conn: Any, tables: Iterable[str]) -> list[Refusal]: - """What attach's own role lacks (the guarded tables' owner roles are `missing`'s - `owner` defects), and a server that allows prepared transactions.""" +async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: + """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` + read: its grants, `SELECT` on every org table, and membership in the owner role of every + table it gives triggers (or the trigger pass could not alter it); and a server that + allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] - for table in tables: - if not await conn.fetchval("SELECT has_table_privilege($1, 'SELECT')", table): - lacks.append(f"SELECT on {table}") + for r in tables: + if not await conn.fetchval( + "SELECT has_table_privilege($1, 'SELECT')", f"loom.{r['relname']}" + ): + lacks.append(f"SELECT on loom.{r['relname']}") + planned = set(cell_triggers.planned_names(tables)) + for r in tables: + if r["relname"] in planned and not r["owned"]: + lacks.append(f"membership in the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -400,51 +430,66 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield - by one try just past `deadlock_timeout`. Then `missing`: any defect refuses.""" - from server.core.schema.pg_migrate import _LOCK_KEY - + by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. + Then `missing`: any defect refuses.""" deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " "FROM pg_settings WHERE name = 'deadlock_timeout'" ) ) - out: list[Refusal] = [] - for table in await cell_triggers.planned_tables(conn): - yielded = False - while True: - lock_timeout = cell_triggers.LOCK_TIMEOUT - holders = await cell_triggers.holders(conn, table) - if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_timeout, yielded = f"{deadlock_ms + 500}ms", True - done = await _ensure_one(conn, table, lock_timeout, _LOCK_KEY) - if done: - break - if not await wait.pause(): - out.append(_busy_refusal(table, await cell_triggers.holders(conn, table))) - break - log(f"loom.{table}: its lock is busy; retrying") - if out: - return out + with wait.counting(): + for table in await cell_triggers.planned_tables(conn): + if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] -async def _ensure_one(conn: Any, table: str, lock_timeout: str, key: int) -> bool: - await conn.execute(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'") +async def _ensure_table( + conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] +) -> Refusal | None: + yielded = False + while True: + lock_ms = 100 + holders = await cell_triggers.holders(conn, table) + if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): + lock_ms, yielded = deadlock_ms + 500, True + lock_ms = wait.bound_ms(lock_ms) + result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) + if result == "applied": + return None + if result == "denied": + return Refusal( + f"postgres loom.{table}", + "attach's role cannot alter it: make the role a member of its owner role", + ) + if not await wait.pause(): + return _busy_refusal(table, await cell_triggers.holders(conn, table)) + log(f"loom.{table}: its lock is busy; retrying") + + +async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: + """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" + from server.core.schema.pg_migrate import _LOCK_KEY + + migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) + if migration_ms <= 0: + return "busy" + await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: - await conn.execute(_MIGRATION_LOCK, key) + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return False # a boot migrating: the next try + return "busy" # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=lock_timeout, attempts=1 + conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) finally: - await conn.execute(_MIGRATION_UNLOCK, key) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -487,49 +532,56 @@ async def commit_enforced( log: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_TIMEOUT` each, - within `wait`; `confirm_lock` before each try and each termination (it refuses when - another run holds the attach lock), and `ready` before each try (False skips the try - while a process boots). After a failed try the in-flight mode transaction, found by its - lock, is ended. Raises Refused when `wait` runs out; whether the commit landed is the - durable read's to say.""" - while True: - await confirm_lock() - if await ready(): - try: - async with conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{_MODE_LOCK_TIMEOUT}'") - await conn.execute(f"SET LOCAL statement_timeout = '{_MODE_STATEMENT_TIMEOUT}'") - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", f"the mode update read {status}" - ) - ] - ) - return - except Refused: - raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise + session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less + when `wait` has less left), within `wait`; `confirm_lock` before each try and each + termination (it refuses when another run holds the attach lock), and `ready` before + each try (False skips the try while a process boots). After a failed try the in-flight + mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; + whether the commit landed is the durable read's to say.""" + with wait.counting(): + while True: + await confirm_lock() + lock_ms = wait.bound_ms(_MODE_LOCK_MS) + if lock_ms > 0 and await ready(): + if await _mode_try(conn, lock_ms): + return log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) - if not await wait.pause(): - raise Refused( - [ - Refusal( - "postgres loom.tenancy_mode", - "its lock was not taken within --wait: a long transaction reads it " - "FOR SHARE; rerun attach", - ) - ] + await end_mode_transactions(conn, confirm_lock) + if not await wait.pause(): + raise Refused( + [ + Refusal( + "postgres loom.tenancy_mode", + "its lock was not taken within --wait: a long transaction reads " + "it FOR SHARE; rerun attach", + ) + ] + ) + + +async def _mode_try(conn: Any, lock_ms: int) -> bool: + """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" + try: + async with conn.transaction(): + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + return True + except Refused: + raise + except Exception as e: + if getattr(e, "sqlstate", None) not in _TIMED_OUT: + raise + return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: @@ -626,6 +678,11 @@ async def wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: """Wait until no backend of the cell's database holds an xid or a snapshot older than `xmax`; Refused at once on a prepared transaction or an unreadable row, and past `wait` naming each backend waited on and each other database's horizon holder.""" + with wait.counting(): + await _wait_transactions(conn, xmax, wait) + + +async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: while True: if refusals := await prepared_refusals(conn): raise Refused(refusals) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 9c2c9755ae..516dee4cc1 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any +from typing import Any, Literal from server.core import cell, pg, readiness @@ -92,6 +92,9 @@ AND l.mode NOT IN ('AccessShareLock', 'RowShareLock') """ +# What a table's DDL came to (`ensure_table`, `_run_ddl`). +Applied = Literal["applied", "busy", "denied"] + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -242,7 +245,12 @@ async def ensure(conn: Any) -> None: async def planned_tables(conn: Any) -> list[str]: """The tables `ensure_table` gives triggers: the guarded org tables and `loom.tenancy`.""" - return [r["relname"] for r, _ in _planned(await conn.fetch(ORG_TABLES))] + return planned_names(await conn.fetch(ORG_TABLES)) + + +def planned_names(rows: list[Any]) -> list[str]: + """`planned_tables` from rows `ORG_TABLES` read.""" + return [r["relname"] for r, _ in _planned(rows)] async def ensure_table( @@ -252,14 +260,14 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: +) -> Applied: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function - passes every row and a boot never creates them. False when the DDL was not applied (its - lock not taken in `attempts` tries of `lock_timeout`, a table this role cannot alter); - True when it was, or nothing was needed. Raises for a table outside the planned set, or - in another state.""" + passes every row and a boot never creates them. `applied` when the DDL ran or nothing + was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, + `denied` when this role cannot alter the table. Raises for a table outside the planned + set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -270,7 +278,9 @@ async def ensure_table( raise ValueError(f"loom.{table} carries no cell triggers") ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) - return not sqls or await _run_ddl(conn, table, sqls, lock_timeout, attempts) + if not sqls: + return "applied" + return await _run_ddl(conn, table, sqls, lock_timeout, attempts) def _table_statements( @@ -293,10 +303,11 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> bool: - """A table's DDL in one transaction, LOCK_ATTEMPTS short tries; whether it applied. The - boot serves past the two causes cells.md names (its lock not taken, a table this role - cannot alter); any other error fails it, as a failed read of the state does.""" +) -> Applied: + """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the + two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: + a table this role cannot alter), which a boot serves past; any other error fails it, as + a failed read of the state does.""" import asyncpg for attempt in range(attempts): @@ -305,12 +316,12 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return True + return "applied" except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return False + return "denied" except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -323,7 +334,7 @@ async def _run_ddl( "; ".join(sqls), await _holders(conn, table), ) - return False + return "busy" async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 406d277646..9bfe3fe9bf 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -633,3 +633,38 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( + cell_stores, +) -> None: + from server.core import cell_triggers + + await _prepare(cell_stores, _Network()) + role = f"it_o_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA loom TO {role}" + ) + await conn.execute(f"SET ROLE {role}") + rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) + refusals = await cell_attach_pg.own_role_refusals(conn, rows) + lacking = [r.reason for r in refusals if "owner role" in r.reason] + assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert any("loom.scope_generations" in r for r in lacking) + # The trigger pass tells the cause apart: denied, not a busy lock. + assert ( + await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + == "denied" + ) + await conn.execute("RESET ROLE") + assert await _mode() == "recording" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index a47f4baec5..b2fb66a467 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -721,15 +721,16 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a async with _cell("recording") as (dsn, conn): with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - assert await cell_triggers.ensure_table(conn, "tenancy", recording_first=True) + ensure = cell_triggers.ensure_table + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert await ensure(conn, "tenancy", recording_first=True) == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) + assert await ensure(conn, "scope_generations", recording_first=True) == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 1c725205e6..b57a40eb57 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -222,12 +222,72 @@ async def forever() -> list[beats.Beat]: @pytest.mark.asyncio -async def test_wait_counts_only_its_own_pauses(clock: _Clock) -> None: - wait = apg.Wait(2, clock) - clock.now += 100 # a scan's time - assert await wait.pause() and await wait.pause() - assert not await wait.pause() - assert wait.spent == 2.0 +async def test_wait_is_wall_time_lock_tries_included_and_no_scan(clock: _Clock) -> None: + """Each mode-commit try waits its lock timeout before it times out: the tries and the + pauses between them end within --wait, and time outside the wait is never charged.""" + asked: list[str] = [] + + def lock(sql: str, *_a: Any) -> str: + if "lock_timeout" in sql: + asked.append(sql) + clock.now += int(sql.split("'")[1].removesuffix("ms")) / 1000 + raise _Timeout() + return "OK" + + conn = FakeConn( + [ + ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + + async def nothing() -> None: + pass + + async def ready() -> bool: + return True + + wait = apg.Wait(4, clock) + clock.now += 100 # a scan before the wait + with pytest.raises(Refused, match="not taken within --wait"): + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + assert clock.now - 100 <= 4 and wait.spent <= 4 + # A 2 s try, a 1 s pause, then a try that waits only the second left. + assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] + + +@pytest.mark.asyncio +async def test_the_trigger_pass_refuses_at_once_a_table_it_cannot_alter( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def ensure_table(*_a: Any, **_kw: Any) -> str: + return "denied" + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", holders) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn([("deadlock_timeout", 1000)]) + wait = apg.Wait(60, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert refusal.subject == "postgres loom.scope_generations" + assert "cannot alter it" in refusal.reason and wait.spent < 1 + + async def busy(*_a: Any, **_kw: Any) -> str: + clock.now += 0.1 + return "busy" + + monkeypatch.setattr(cell_triggers, "ensure_table", busy) + wait = apg.Wait(3, clock) + (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + assert "not taken within --wait" in refusal.reason and wait.spent <= 3 # -- the rescan's watch ---------------------------------------------------------------- @@ -272,9 +332,9 @@ def test_the_watch_refuses_up_front_with_a_very_long_heartbeat( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(settings, "instance_heartbeat_seconds", 1180) - assert beats.Watch(None, "", "first").refusals_up_front() + assert beats.prune_window_refusals() monkeypatch.setattr(settings, "instance_heartbeat_seconds", 10) - assert beats.Watch(None, "", "first").refusals_up_front() == [] + assert beats.prune_window_refusals() == [] # -- batches, ordering, arguments -------------------------------------------------------- @@ -491,10 +551,19 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ("max_prepared_transactions", 5), ] ) - refusals = await apg.own_role_refusals(conn, ["loom.memory_current", "loom.tenancy"]) + tables = [ + {"relname": "memory_current", "relkind": "r", "owned": True}, + # Its grants are held, but not its owner role's membership: the trigger pass could + # not alter it, so it refuses before any write. + {"relname": "scope_generations", "relkind": "r", "owned": False}, + {"relname": "tenancy", "relkind": "r", "owned": True}, + ] + refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks membership in pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", + "postgres role attacher: attach's role lacks membership in the owner role of " + "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", ] @@ -674,7 +743,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] assert await run.done() state[0] = "recording" - with pytest.raises(Refused, match="run loom cell restore"): + with pytest.raises(Refused, match="attach lowers and repairs neither"): await run.done() @@ -697,3 +766,38 @@ async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: await run.final_checks(beats.Watch(None, "", "first")) run.begun_keys_fp = "after" await run.final_checks(beats.Watch(None, "", "first")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [Refused([Refusal("postgres", "cut")]), OSError("gone")]) +async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outcome_is_unknown( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: Exception +) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + async def landed(*_a: Any) -> None: + pass # the commit landed; its answer may not have + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def run(self: Any) -> int: + self.row = {"deployment_id": "dep-1"} + await self.commit_mode() + raise failure # what stops the run before the durable read finishes + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "commit_enforced", landed) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "run", run) + url = httpx.URL("http://127.0.0.1:7900") + assert await cell_attach.attach(None, _args(), url) == 1 + err = capsys.readouterr().err + assert "the mode commit's outcome is unknown: rerun attach" in err + assert "stays recording" not in err From 5884b04347fd9ec9d839bfbe8c9f780fef3b7607 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:17:07 +0000 Subject: [PATCH 024/181] Doctor's stray-org read names a table it cannot read and goes on, and the attach docs describe one step numbering and today's procedure --stray-orgs labels only a spent budget as tables left unchecked; any other refusal names its table and the read continues. A failed catalog read reports both of attach's Postgres checks as not read. The runbook and doctor describe removing a stray or over-long org by hand, and name no command that does not exist yet. cells.md labels the first attach's list with the design's step names, so step 1 to step 4 mean the same everywhere, rewrites the stuck-process failure row and the planned tests for a mode that only rises, and says --wait is wall time. The trigger-cost harness uses cell_triggers.create_statement and deletes the rows its inserts wrote. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 21 +- docs/architecture/cells.md | 405 +++++++++++++------------- docs/operations/cell-attach.md | 19 +- server/cli/cell_attach.py | 41 +-- server/cli/doctor_cell.py | 36 ++- server/core/cell_triggers.py | 5 + tests/integration/test_cell_attach.py | 20 ++ tests/unit/test_cell_attach.py | 27 ++ 8 files changed, 325 insertions(+), 249 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e90fbb32fa..4363a810d6 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,9 +1,10 @@ """The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays). Run it against an attached cell's server, which must not serve anything else, -with a writer token for a placed org; it drops and recreates the cell's org triggers, so -never against a deployment in use. Needs no provider. +guard stays), deleting the rows its inserts wrote at the end. Run it against an attached +cell's server, which must not serve anything else, with a writer token for a placed org; +it drops and recreates the cell's org triggers, so never against a deployment in use. +Needs no provider. PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ --base-url http://127.0.0.1:7932 --pg-dsn \\ @@ -36,7 +37,7 @@ async def _triggers(dsn: str, on: bool) -> None: for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: - await conn.execute(cell_triggers._create(f'loom."{table}"', name)) + await conn.execute(cell_triggers.create_statement(table, name)) enabled = await conn.fetchval( "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", list(cell_triggers.ORG_TRIGGERS), @@ -111,6 +112,18 @@ async def main() -> None: ) finally: await _triggers(args.pg_dsn, True) + await _clean(args) + + +async def _clean(args: argparse.Namespace) -> None: + """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" + conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) + try: + await conn.execute( + "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + ) + finally: + await conn.close() if __name__ == "__main__": diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 46e1439538..3ea7ddab80 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -682,7 +682,8 @@ mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_ this database (a pid captured at connect is wrong behind a connection path that swaps backends), and waits until the lock clears. Whether the commit landed is decided by a durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell), steps 6 and 7). +it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the +durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -881,14 +882,13 @@ read-only scope, lists any tenancy row the directory does not place. yet, a later cell's configured orgs at attach and a changed one at `loom cell update` included, passes the gate, a closed cell and `placements_max` included, so a later cell closed or full before attach refuses the attach, naming the org. -- **The first cell records until `enforced` as a single deployment does.** In - `recording`, which lasts until step 3 writes `enforced` to the cell's mode (and again - whenever step 3 or step 4 returns the cell to it), a verified request for a new org - writes a tenancy row with no placement call, so the router's rate, `placements_max` - and `loom cell limit --closed` do not apply: new orgs arrive as they do on a single - deployment, bounded by its own signup, issuer and JIT settings, and step 2 or step 4 - places them unrefused. `loom doctor` lists the recorded orgs that hold no data, so the - operator sees what attach will place. +- **The first cell records until `enforced` as a single deployment does.** In `recording`, + which lasts until step 3 writes `enforced` to the cell's mode (the mode only rises after + it), a verified request for a new org writes a tenancy row with no placement call, so the + router's rate, `placements_max` and `loom cell limit --closed` do not apply: new orgs + arrive as they do on a single deployment, bounded by its own signup, issuer and JIT + settings, and step 2 or step 4 places them unrefused. `loom doctor` lists the recorded + orgs that hold no data, so the operator sees what attach will place. - **Configured orgs are placed at attach.** The cell's `oidc_jit_org`, the `self_signup_org` of a cell with signup on, and its code orgs (`LOOM_CODE_ORG` and the org of each `LOOM_CODE_REPOS_JSON` entry the code config keeps) and tracker org are @@ -986,8 +986,8 @@ The first attach (`loom cell attach`, its procedure on [attaching the first cell](../operations/cell-attach.md)) turns the existing deployment into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at `attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an -advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare and -`loom cell update` take the same lock), after checking that the session is one direct +advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare +takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked writer's. A lock found lost between steps is taken again (refusing when another run holds @@ -1034,74 +1034,76 @@ It refuses, before any write, naming each: `server/cli/cell_scan.py`), and a settings profile or quota that caps what attach's user reads. -Then, in order: - -1. `attach/begin`, on every run, before any long step: the router probes the stores - attach wrote its nonces to, checks the upstream, and writes the fingerprints and - `keys_fp`; a directory restored to before it is written again. -2. Read the mode: `recording`, or `enforced` on a resume, which goes from the triggers, - through `attach/enforced` again, to the durable read. -3. Create the triggers itself, on every run, before step 2's scan and again just before the - mode commit (so a table a migration added during the scan is guarded from the commit - on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock - and retried a second apart within `--wait`; an autovacuum holding it gets one try just - past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then - `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the - triggers, which pass every row there. -4. In `recording`, the scan: one statement per table, no long transaction (step 4 reads - every source again): the Postgres tables (`loom.tenancy` among them) by skip scans on an - `org`-led btree index, then the ClickHouse tables, each flushed and synced first, with - the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not - read: a log row keeps the org a caller typed before any check, so placing a log-only - org would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such - orgs. An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row - is written; a name no request can carry is skipped and named. Then the tenancy rows of - the orgs found, on a marked session, and no placement: unscoped callers are answered - from the tenancy rows, and placing now would move the point of no return earlier. -5. The pre-commit checks: the heartbeats, the tenancy-privilege and unmarked-session - checks and the standbys again, and the scan's elapsed time within half of - `--scan-budget`, since step 4 reads the same tables again. Each refuses in - `recording`, which can still be backed out. -6. The mode commit: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on - `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock - (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database - alone) and ended. From the commit on the mode only rises: every later refusal leaves - the cell `enforced` and `attaching`, and a rerun resumes from the durable read. -7. The durable read, in every run that reads `enforced`: the mode and the `first` row read - in a transaction that emits a transactional logical message, whose commit waits for - the synchronous standbys, with the standbys checked attached just before and after. - Then `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED - writer that read `recording` holds an xid below `X`, and a REPEATABLE READ one a - snapshot older than `X`: the trigger functions are `VOLATILE`, and the message's xid is - assigned after the mode commit. -8. The heartbeats again: a process with `LOOM_CELL` unset or another cell, without the - keys, unmarked, or with no router where it serves, refuses at once. No process depends - on its cached mode (a request records a new org through `tenancy._RECORD`, which reads - the mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. -9. Step 4: wait, within `--wait`, while any backend of the cell's database but the polling - one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` - (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach - refuses, naming each, and each backend of another database holding the horizon. A - prepared transaction, or a row of the cell's database whose state is hidden, refuses - at once. Then the cheap checks (the writer check, `missing`, the Postgres checks, the - heartbeats with every stopped process with `LOOM_CELL` unset aged out, the standbys); - the rescan of every source in full, Postgres before ClickHouse (the outbox and delivery - tables move rows to ClickHouse), and the queue and drain tables and the tables a view - reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, - in `attach/place` batches; the writer check again; and the final checks, under a floor - of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): - the heartbeats, read through the rescan every half of the time the leader keeps a - stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` - unset during it, `loom.cell_self` equal to the directory's fingerprints, one `keys_fp` - equal to the one `begin` wrote, the mode `enforced`; then `attach/serving`. - -`--wait` (default 10 minutes) is shared by the waits alone; `--scan-budget` (default 2 -hours) bounds each full scan. `--dry-run` runs the refusals and the scan, prints what would -be placed and skipped and the scan's time, and writes nothing; the router's checks at -`attach/begin` do not run. On a `serving` row attach reads the cell's mode: `enforced` is -done (exit 0), anything else refuses, naming `loom cell restore`. +Then, in order (the step numbers are the ones the rest of this page uses): + +- **`attach/begin`**, on every run, before any long step: the router probes the stores + attach wrote its nonces to, checks the upstream, and writes the fingerprints and + `keys_fp`; a directory restored to before it is written again. +- **Step 1, the mode**: `recording`, or `enforced` on a resume, which goes from the + triggers, through `attach/enforced` again, to the durable read. +- **The triggers**: created by attach itself, on every run, before step 2 and again just + before step 3 (so a table a migration added during the scan is guarded from the commit + on): one table at a time under the migration lock, each DDL waiting 100 ms for its lock + and retried a second apart within `--wait`; an autovacuum holding it gets one try just + past `deadlock_timeout`, so it yields, and an anti-wraparound vacuum is named. Then + `cell_triggers.missing`; any defect refuses. A refused attach in `recording` leaves the + triggers, which pass every row there. +- **Step 2, the scan**, in `recording`: one statement per table, no long transaction (step 4 + reads every source again): the Postgres tables (`loom.tenancy` among them) by skip scans + on an `org`-led btree index, then the ClickHouse tables, each flushed and synced first, + with the configured orgs of both fingerprints. Log tables (`cell_scan.LOG_TABLES`) are not + read: a log row keeps the org a caller typed before any check, so placing a log-only org + would let anyone pin an org name to the cell; `loom doctor --stray-orgs` names such orgs. + An org longer than `cell.ORG_MAX_BYTES` refuses, naming it, before any tenancy row is + written; a name no request can carry is skipped and named. Then the tenancy rows of the + orgs found, on a marked session, and no placement: unscoped callers are answered from the + tenancy rows, and placing now would move the point of no return earlier. +- **The pre-commit checks**: the heartbeats, the tenancy-privilege and unmarked-session + checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, + since step 4 reads the same tables again. Each refuses in `recording`, which can still be + backed out. +- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE + lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the + cheap heartbeat refusals run before each try (a booting process skips the try). After a + failed try, the mode transaction an earlier run left in flight is found by its lock (only + attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) + and ended. From the commit on the mode only rises: every later refusal leaves the cell + `enforced` and `attaching`, and a rerun resumes from the durable read. +- **The durable read**, in every run that reads `enforced`: the mode and the `first` row + read in a transaction that emits a transactional logical message, whose commit waits for + the synchronous standbys, with the standbys checked attached just before and after. Then + `X`, the snapshot's xmax, which must exceed that message's xid. A READ COMMITTED writer + that read `recording` holds an xid below `X`, and a REPEATABLE READ one a snapshot older + than `X`: the trigger functions are `VOLATILE`, and the message's xid is assigned after + the mode commit. +- **The heartbeats** again: a process with `LOOM_CELL` unset or another cell, without the + keys, unmarked, or with no router where it serves, refuses at once. No process depends on + its cached mode (a request records a new org through `tenancy._RECORD`, which reads the + mode `FOR SHARE`), so neither a cache lifetime nor a request deadline is waited out. +- **Step 4**: wait, within `--wait`, while any backend of the cell's database but the + polling one has an xid older than `X`, or any but an autovacuum a snapshot older than `X` + (compared through `age()`, exactly `< X` and wraparound-safe); past `--wait` attach + refuses, naming each, and each backend of another database holding the horizon. A prepared + transaction, or a row of the cell's database whose state is hidden, refuses at once. Then + the cheap checks (the writer check, `missing`, the Postgres checks, the heartbeats with + every stopped process with `LOOM_CELL` unset aged out, the standbys); the rescan of every + source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to + ClickHouse), and the queue and drain tables and the tables a view reads before the others + (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; + the writer check again; and the final checks, under a floor of their own (the larger of + what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the + rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), + refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to + the directory's fingerprints, one `keys_fp` equal to the one `begin` wrote, the mode + `enforced`; then `attach/serving`. + +`--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try +included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each +full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints +what would be placed and skipped and the scan's time, and writes nothing; the router's +checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: +`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's +Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1297,13 +1299,13 @@ the directory row. | A first cell restored to before its prepare, not yet attached | at processes with `LOOM_CELL` set, every request but health probes and the probe route answers 503 until prepare reruns | | The directory lost | every cell serves the orgs it holds; placements and uncached lookups answer 503, directory commands refuse, until it is back | | A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | -| A process stuck during first attach | the attach refuses after `--wait` and the cell returns to `recording` | +| A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | | A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | | Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | | A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | | A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | | A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | -| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until the stray-org erase command removes them; `loom doctor --stray-orgs` names it | +| An org found only in a log table of the first cell | never placed; once it is placed in another cell, its log rows here (`agent_events` identities among them) are outside its erasure there until they are removed by hand (the runbook's procedure); `loom doctor --stray-orgs` names it | | An org longer than `cell.ORG_MAX_BYTES` written before the limit | the first attach refuses in its scan, naming it, until the runbook's manual procedure removes it; one first found in step 4 is skipped and named | | A writer, grant, role setting or subscription added after `serving` | attach's checks ran once and pass it; `loom doctor` warns on the Postgres ones on every run | | `LOOM_SERVES_TRAFFIC=false` on a pod a load balancer routes to | attach does not require its router answer; its new orgs answer 503 | @@ -1474,126 +1476,117 @@ An integration test with two cells and a router in CI's Postgres job: - every route, the batch entries of MCP and every CLI command included, refuses another cell's org and writes nothing in its Postgres or ClickHouse (the route, op-field and CLI censuses cover the rest); -- the first attach places every org with a row in any table, an org first seen during - the attach is placed in step 4, and so is one a direct Postgres insert wrote between - step 2's scan and `enforced`, a ClickHouse row an older CLI wrote then and a - bucket object catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` - of failed reads of its own rows or mode is gated and serves again once a read lands; a - pod with `LOOM_CELL` unset is unready, naming the setting, answers 503 to a bearer - request, passes its health probe, answers the probe route and runs no worker on an - attached cell and on a later cell between prepare and attach, keeps serving on a first - cell in `recording`, and keeps the last mode it read through a failed read, - `recording` included; a CLI command - declaring an org with it unset is refused on an `enforced` or `partial` cell, an - org-free one (a token revocation) runs, also with Postgres down on a deployment never - prepared, and a `loom cell` store command refuses; `loom cell limit --closed` succeeds - with the cell's Postgres down; after `loom cell credential --cell --next` both - credentials place and read state, `--finish` refuses the old credential, for a cell - finished before its pods hold the new one the router refuses its calls with 401, its - request answers 503 writing nothing, and it recovers by recording the old hash again, - `--revert` drops a pending `next`, which is then refused, and the old credential keeps - placing, and a second `--next` while one is pending and a hash another cell records as - current or next are refused; `loom cell limit` and `loom cell credential --next`, - `--finish` and `--revert` refuse while the directory is unavailable; concurrent - placements at one under `placements_max` across two router replicas write one; a CLI - command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a - first cell once it is `enforced` (`attaching` included) and on a later cell, and on a - first cell in `recording` runs and records the org; attach with `placements_max` below - the orgs the first cell holds places them all, and a later cell's attach past its - quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step - 2 and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's - mode, a process whose cache still reads `recording` sends a new org's request to the - placement call, which a closed cell answers 503; a CLI command whose insert finds - `enforced` refuses, naming the org, and makes no placement call; after step 3's - refused wait or step 4's recheck returns the cell to `recording`, a new org is - recorded with no placement call, and one at a process whose cache still reads - `enforced` places through the gate until the cache expires; each ordering of attach's - write of the mode with a recording insert holds as stated above (running, starting - during the write, a stream of overlapping inserts each shorter than a try, with attach - setting `enforced` within a try or two, held across several tries, held until too - little of `--wait` is left, a stalled attach, a commit cut off by its timeout, a - commit whose connection is lost while it waits for the standby, read back only once - that backend has gone and written again durably if it reads `enforced`); a - `REPEATABLE READ` transaction whose first statement ran before `enforced` committed, - on attach's role or a second role, holds step 4 until it ends and its write of an - unheld org is then placed, one whose first statement ran after is refused by the - trigger, a transaction that starts after attach takes the lock but before its commit - is visible holds step 4, a transaction opened in `recording` that outlives one run's - `--wait` holds the rerun's step 4, after a refusal and after `loom cell restore` - raised the mode, an autovacuum worker on a loom table does not hold step 4 and a - background worker with a transaction open since `recording` does, a prepared - transaction (after a restart with `max_prepared_transactions` raised during the wait) - makes step 4 refuse at once, naming it, with the mode still `enforced` and a rerun - refused until it is resolved and the setting is back to 0, a write transaction open in - a second database on the same server holds step 4 and is named, a long read on the - synchronous standby, or on a second standby, with `hot_standby_feedback` on does not - hold it, nor does a base backup running across it, a write held open on a +- the first attach places every org with a row in any table, an org first seen during the + attach is placed in step 4, and so is one a direct Postgres insert wrote between step 2's + scan and `enforced`, a ClickHouse row an older CLI wrote then and a bucket object + catalogued then; a process past `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` of failed reads of + its own rows or mode is gated and serves again once a read lands; a pod with `LOOM_CELL` + unset is unready, naming the setting, answers 503 to a bearer request, passes its health + probe, answers the probe route and runs no worker on an attached cell and on a later cell + between prepare and attach, keeps serving on a first cell in `recording`, and keeps the + last mode it read through a failed read, `recording` included; a CLI command declaring an + org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token + revocation) runs, also with Postgres down on a deployment never prepared, and a `loom +cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres + down; after `loom cell credential --cell --next` both credentials place and read + state, `--finish` refuses the old credential, for a cell finished before its pods hold the + new one the router refuses its calls with 401, its request answers 503 writing nothing, + and it recovers by recording the old hash again, `--revert` drops a pending `next`, which + is then refused, and the old credential keeps placing, and a second `--next` while one is + pending and a hash another cell records as current or next are refused; `loom cell limit` + and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is + unavailable; concurrent placements at one under `placements_max` across two router + replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set + refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a + later cell, and on a first cell in `recording` runs and records the org; attach with + `placements_max` below the orgs the first cell holds places them all, and a later cell's + attach past its quota or closed refuses, naming the org; a first cell with `--closed` and + `placements_max` set before step 2 attaches, and an org a request records between step 2 + and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a + process whose cache still reads `recording` sends a new org's request to the placement + call, which a closed cell answers 503; a CLI command whose insert finds `enforced` + refuses, naming the org, and makes no placement call; a refusal after step 3's write + leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; + each ordering of attach's write of the mode with a recording insert holds as stated above + (running, starting during the write, a stream of overlapping inserts each shorter than a + try, with attach setting `enforced` within a try or two, held across several tries, held + past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut + off by its timeout, and a commit whose connection is lost while it waits for the standby, + which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran + before `enforced` committed, on attach's role or a second role, holds step 4 until it ends + and its write of an unheld org is then placed, one whose first statement ran after is + refused by the trigger, a transaction that starts after attach takes the lock but before + its commit is visible holds step 4, a transaction opened in `recording` that outlives one + run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom + table does not hold step 4 and a background worker with a transaction open since + `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` + raised during the wait) makes step 4 refuse at once, naming it, with the mode still + `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write + transaction open in a second database on the same server holds step 4 and is named, a long + read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on + does not hold it, nor does a base backup running across it, a write held open on a `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a - long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database - past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the - first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` - above 0 refuses before step 1 with the mode still `recording`, while a later cell's - attach does not require either; a missing or disabled org trigger is refused by attach - and named by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode - row was deleted is refused and the state named by `loom doctor`; on that cell a pod - with `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an - org with it unset is refused, as a pod is; updating, deleting or truncating the - `first` row on an `enforced` cell is refused, also under - `session_replication_role = replica`, and a missing or disabled `first`-row trigger is - refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set - on a cell whose mode row is removed answers 503 to a bearer request and runs no worker - pass, and serves again after a rerun of `loom cell prepare` writes the row back while - it runs; a cell the directory registers with no state, tenancy or own row is named by - `loom doctor`, its processes with `LOOM_CELL` set stay unready, a process with it - unset serves as a single deployment, and `loom cell restore` writes both rows back, an - attached cell then serving again and an unattached one after a prepare rerun; a cell - with both rows absent beside a tenancy row, and one beside only its own row, has the - function refuse an unheld org (a boot creates no trigger there), every process gated - and `loom doctor` naming it, and a prepare rerun writes the rows back; a Postgres-only - restore of an attached first cell to before its prepare is named by `loom doctor` and - closed by `loom cell restore`; a prepare rerun during a pending `loom cell update` on - an attached cell, with processes on both values and `loom.cell_self` lost, writes both - values back and keeps both sets of processes serving, and one before attach refuses - the two configurations; a prepare rerun on an attached cell whose `current` is missing - and whose pods' configuration differs from the directory row refuses, and one before - attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a - credential hash that is neither the row's `current` nor its `next` is refused; a - prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode - row back; a cell with a mode row and no `first` row has the function refuse an unheld - org (a boot creates no trigger there), every process gated and `loom doctor` naming - it, and a prepare rerun writes the row back; a later cell whose mode row is set to - `recording` records nothing, its trigger refuses an unheld org, a process with - `LOOM_CELL` set there serves as in `enforced`, a pod with `LOOM_CELL` unset there is - unready and runs no worker, `loom cell prepare` names the row, a CLI command declaring - an org with `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses - its state read; a build N-1 process's `memory.set` and delivery pass on a - never-prepared deployment after N's migration succeed and write no tenancy row, and on - the first cell in `recording` a boot creates no org trigger, and one created by hand - reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), - an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused - and one that hits an existing row of it is not, a marked session's every write passes, - and the triggers on an `enforced` cell cover every table erasure classifies with an - `org` column but the three exclusions; and an org-keyed write while attach holds the - mode table's lock returns at once, with new-org requests answering within their - deadline and mode-cache reads returning at once throughout; `loom cell restore` after - a refusal at step 3's lock leaves the cell `enforced`, and the resumed attach runs - from step 3's wait; a `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at - boot and named by `loom doctor`; `loom cell credential new` prints a credential of - that form and its SHA-256, with the directory and the stores unreachable; prepare and - `--next` refuse an argument that is not 64 lowercase hex characters, the credential - and its bare 128-character secret included, storing nothing; orgs a first cell records - in `recording` with no data are listed by `loom doctor`; a placement flood with no - quota set is held to the rate limit at each router replica, `loom doctor` names the - cell as one with no quota, and a call refused past it at one replica answers 503 with - a `Retry-After` of at most 60 s, after which its retry at that replica, with no other - call of the cell admitted between, is admitted; a cell's first prepare, with no - directory row yet, passes the deployment-id check; a pod or CLI that lost - `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the authority check; - the first cell's existing tokens, refresh tokens and OAuth clients keep working - through the cutover, a logout through the shared host revokes its refresh token; + transaction on one, opened in `recording` and writing nothing before `enforced`, a long + logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past + `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first + cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 + refuses before step 1 with the mode still `recording`, while a later cell's attach does + not require either; a missing or disabled org trigger is refused by attach and named by + `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted + is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset + is unready and runs no worker, and a CLI command declaring an org with it unset is + refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` + cell is refused, also under `session_replication_role = replica`, and a missing or + disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer + request and runs no worker pass, and serves again after a rerun of `loom cell prepare` + writes the row back while it runs; a cell the directory registers with no state, tenancy + or own row is named by `loom doctor`, its processes with `LOOM_CELL` set stay unready, a + process with it unset serves as a single deployment, and `loom cell restore` writes both + rows back, an attached cell then serving again and an unattached one after a prepare + rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, + has the function refuse an unheld org (a boot creates no trigger there), every process + gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a + Postgres-only restore of an attached first cell to before its prepare is named by `loom +doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell +update` on an attached cell, with processes on both values and `loom.cell_self` lost, + writes both values back and keeps both sets of processes serving, and one before attach + refuses the two configurations; a prepare rerun on an attached cell whose `current` is + missing and whose pods' configuration differs from the directory row refuses, and one + before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given + a credential hash that is neither the row's `current` nor its `next` is refused; a prepare + rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a + cell with a mode row and no `first` row has the function refuse an unheld org (a boot + creates no trigger there), every process gated and `loom doctor` naming it, and a prepare + rerun writes the row back; a later cell whose mode row is set to `recording` records + nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom +cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` + set refuses, naming it, and the router refuses its state read; a build N-1 process's + `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed + and write no tenancy row, and on the first cell in `recording` a boot creates no org + trigger, and one created by hand reads the state and looks nothing up, a write under + `session_replication_role = replica` for an unheld org is skipped (origin triggers), an + `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one + that hits an existing row of it is not, a marked session's every write passes, and the + triggers on an `enforced` cell cover every table erasure classifies with an `org` column + but the three exclusions; and an org-keyed write while attach holds the mode table's lock + returns at once, with new-org requests answering within their deadline and mode-cache + reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` + with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom +doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with + the directory and the stores unreachable; prepare and `--next` refuse an argument that is + not 64 lowercase hex characters, the credential and its bare 128-character secret + included, storing nothing; orgs a first cell records in `recording` with no data are + listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at + each router replica, `loom doctor` names the cell as one with no quota, and a call refused + past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its + retry at that replica, with no other call of the cell admitted between, is admitted; a + cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or + CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the + authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep + working through the cutover, a logout through the shared host revokes its refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the @@ -1607,12 +1600,12 @@ An integration test with two cells and a router in CI's Postgres job: application ends the person there, a platform-admin bot still reaches a cell whose SCIM deactivated everyone, and attach refuses a cell with unprovisioned live subjects or sharing another's OIDC client; -- an attach stopped in step 3 after it set `enforced`, before the wait passed, resumes - with the wait before the SCIM precondition, and one refused by the SCIM precondition - resumes from the wait too, the mode still `enforced`; a later attach refused before - its directory transaction leaves `attach_started_at` empty, so a rerun checks a new - cell's refusals again, and a resumed first attach finds its configured orgs placed in - this cell and carries on; +- an attach stopped after step 3's commit landed, before the durable read, says the commit's + outcome is unknown, and its rerun resumes by the mode it reads; one refused after the + durable read resumes from it, the mode still `enforced`; a later attach refused before its + directory transaction leaves `attach_started_at` empty, so a rerun checks a new cell's + refusals again, and a resumed first attach finds its configured orgs placed in this cell + and carries on; - a cell restored to before the first attach's `enforced` gets its mode back from `loom cell restore`; one whose `loom cell restore` was skipped answers 503 for an org it does not hold, another cell's org included, writes no tenancy row for it, and is diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 12a0cbe247..c51541d06d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -105,17 +105,16 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse always goes forward. From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`) until `loom cell place` is available; an -ordinary mint scoped to its org places the org on its first request. A direct call of the -router's operator place route, or a live placement with the cell's credential, cannot read -the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's -credential is the servers' alone. +routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places +the org on its first request. A direct call of the router's operator place route, or a live +placement with the cell's credential, cannot read the cell's Postgres and is accepted once +the directory mode is `enforced`; the cell's credential is the servers' alone. ## Orgs attach does not place - **An org longer than 2048 bytes** refuses step 2, naming it, before any tenancy row is - written. One first found in step 4 is skipped and named. Remove such an org by hand - until the stray-org erase command exists, in this order: its bucket objects, from the + written. One first found in step 4 is skipped and named. Remove such an org by hand, in + this order: its bucket objects, from the keys of its rows in `loom.documents`, `loom.document_blobs` and `loom.blob_uploads`, so no memory content is left outside erasure; then its rows in every table with an `org` column of both ClickHouse databases and of Postgres schema `loom`, the erasure ledger tables @@ -128,9 +127,9 @@ credential is the servers' alone. erasure there. `loom doctor --stray-orgs` names the orgs the cell's stores hold without a tenancy row, -smallest tables first within `--stray-orgs-budget` (default 60 s), and every table it left -unchecked. `loom cell place` and `loom cell restore`, which act on these, are not available -yet. +smallest tables first within `--stray-orgs-budget` (default 60 s), every table the budget +left unchecked and every table it could not read. Remove a stray org's rows by the same +procedure as an over-long org's. ## After attach diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ea9cf77d35..52abea19c6 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -3,26 +3,27 @@ docs/operations/cell-attach.md). A run holds the attach lock on a direct session of the cell's Postgres for its life, and -goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode: - -1. the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; -2. the refusals, before any write: the stores are the registered ones, the session is - direct and marked, the cell's processes (their heartbeats), the roles, settings and - subscriptions that would let an unchecked writer through, the ClickHouse writers, - the table list, durability, attach's own grants; -3. `attach/begin` (the router probes the stores and writes the fingerprints); -4. the mode: `recording`, or `enforced` on a resume; -5. the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; -6. in `recording`, step 2's scan, refusing an org past `cell.ORG_MAX_BYTES`, and the - tenancy rows of the orgs found; the triggers again; the pre-commit checks; -7. step 3: `attach/enforced`, then the mode commit (never on a resume): from it on the mode - only rises, so every later refusal leaves the cell `enforced` and `attaching`; -8. the durable read of `enforced`, the heartbeats, step 4's wait for every transaction - that could have read `recording`, the rescan, the tenancy rows and their placements - (`attach/place`), the final checks and `attach/serving`. - -`--wait` bounds the waits, `--scan-budget` each full scan. `--dry-run` runs the refusals -and the scan and prints what would be placed, writing nothing. +goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. +Steps 1 to 4 are the design's, as cells.md numbers them: + +- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the refusals, before any write: the stores are the registered ones, the session is + direct and marked, the cell's processes (their heartbeats), the roles, settings and + subscriptions that would let an unchecked writer through, the ClickHouse writers, the + table list, durability, attach's own grants; +- `attach/begin` (the router probes the stores and writes the fingerprints); +- step 1, the mode: `recording`, or `enforced` on a resume; +- the triggers (`cell_triggers.ensure_table`), under the migration lock one try at a time; +- in `recording`: step 2, the scan, refusing an org past `cell.ORG_MAX_BYTES`, and the + tenancy rows of the orgs found; the triggers again; the pre-commit checks; step 3, + `attach/enforced` and the mode commit, from which on the mode only rises, so every + later refusal leaves the cell `enforced` and `attaching`; +- the durable read of `enforced`, the heartbeats, and step 4: the wait for every + transaction that could have read `recording`, the rescan, the tenancy rows and their + placements (`attach/place`), the final checks and `attach/serving`. + +`--wait` bounds the waits in wall time, `--scan-budget` each full scan. `--dry-run` runs +the refusals and the scan and prints what would be placed, writing nothing. """ from __future__ import annotations diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index c79723bbbc..9edb8e275d 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -186,7 +186,10 @@ async def _checks(ch: Any, allowed_roles: tuple[str, ...] = ()) -> list[tuple[st try: reads: list[tuple[str, str, str]] = await _bounded(_writer_reads(allowed_roles)) except Exception as e: # noqa: BLE001 — an unread catalog is named, not a failed run - reads = [("cell_tenancy_privilege", WARN, f"not read: {_reason(e)}")] + reads = [ + (name, WARN, f"not read: {_reason(e)}") + for name in ("cell_tenancy_privilege", "cell_unchecked_writers") + ] return _grades(state, marked, marks_all, defects) + reads @@ -264,6 +267,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: budget = cell_scan.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] + refused: list[str] = [] for i, (_n, store, table) in enumerate(work): found = cell_scan.ScanResult() name = table if store == "postgres" else table.canonical @@ -274,9 +278,15 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: else: for raw in await cell_scan.ch_orgs(ch, table, topology, budget): found.add(name, raw) - except Refused: - unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] - break + except Refused as e: + # The budget spent leaves this table and the larger ones unread; any other + # refusal (a table with no org-led index, a replica that cannot sync) names + # this table alone, and the read goes on. + if budget.left() <= 1: + unchecked += [w[2] if w[1] == "postgres" else w[2].canonical for w in work[i:]] + break + refused += [str(r) for r in e.refusals] + continue orgs = sorted(set(found.orgs) - held) + sorted(found.too_long) bad = [m for ms in found.malformed.values() for m in ms] if orgs or bad: @@ -290,13 +300,20 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: or "every org found has a tenancy row", ) ] - if unchecked or problems: + if unchecked: out.append( ( "cell_stray_orgs_unchecked", WARN, - f"not read within --stray-orgs-budget ({budget_s:g} s): " - + ", ".join(unchecked + [str(p) for p in problems]), + f"not read within --stray-orgs-budget ({budget_s:g} s): " + ", ".join(unchecked), + ) + ) + if refused or problems: + out.append( + ( + "cell_stray_orgs_unread", + WARN, + "not read: " + "; ".join(refused + [str(p) for p in problems]), ) ) return out @@ -305,8 +322,9 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: def stray_org_checks(budget_s: float) -> list[tuple[str, str, str]]: """`loom doctor --stray-orgs`: the orgs the cell's stores hold without a tenancy row (log-only orgs, names no request can carry, rows an unchecked writer left), smallest - tables first within one budget, naming every table left unchecked. Never a FAIL: the - stray-org erase command removes what it names.""" + tables first within one budget, naming every table left unchecked and every table it + could not read. Never a FAIL: the operator removes what it names by hand + (docs/operations/cell-attach.md).""" from server.cli._ch import connect_declared, disconnect from server.core.ch import get_client, statement_timeout diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 516dee4cc1..4395138b4d 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -176,6 +176,11 @@ def _function_sql(function: str) -> re.Pattern[str]: ) +def create_statement(table: str, name: str) -> str: + """The CREATE TRIGGER for `name` on `loom.
`, as `ensure` writes it.""" + return _create(f"loom.{_ident(table)}", name) + + def _create(qualified: str, name: str) -> str: """The CREATE TRIGGER for `name` on the table `qualified` names.""" if name == TENANCY_TRIGGER: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 9bfe3fe9bf..e0f96f409f 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -668,3 +668,23 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_stores) -> None: + from server.cli import doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + # An org table with no org-led index: its skip scan refuses, the read goes on. + await pg.execute("CREATE TABLE loom.it_no_org_index (id int PRIMARY KEY, org text)") + await pg.execute("INSERT INTO loom.it_no_org_index VALUES (1, $1)", tag + "unindexed") + await pg.execute( + "INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", tag + "stray" + ) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} + assert tag + "stray" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unread"] + assert grade == "WARN" and "postgres loom.it_no_org_index" in detail + assert "cell_stray_orgs_unchecked" not in lines diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b57a40eb57..857d12dfe0 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -801,3 +801,30 @@ async def run(self: Any) -> int: err = capsys.readouterr().err assert "the mode commit's outcome is unknown: rerun attach" in err assert "stays recording" not in err + + +@pytest.mark.asyncio +async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.cli import doctor_cell + + async def before(_ch: Any) -> str: + return "" + + async def marked() -> tuple[str, bool, list[Any]]: + return "enforced", True, [] + + async def unmarked() -> bool: + return False + + async def unread(_roles: Any) -> Any: + raise OSError("the catalog read failed") + + monkeypatch.setattr(doctor_cell, "_deployment_check", before) + monkeypatch.setattr(doctor_cell, "_marked_session", marked) + monkeypatch.setattr(doctor_cell, "_unmarked_session", unmarked) + monkeypatch.setattr(doctor_cell, "_writer_reads", unread) + lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} + for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): + assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") From 691f4f0eeb11f02b6d31dbc44f49a5d43583536f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 025/181] Attach's closing line names the mode it last read, every DDL try ends inside --wait, and doctor warns on the role it runs as The mode is taken from the first state read, set to unknown only once the router accepted attach/enforced, and set from what the durable read saw; an interrupt prints the mode line and goes on. The trigger DDL's lock bound is taken after the migration lock, from what --wait has left then. An ended mode transaction whose lock does not clear refuses, naming its backend. A busy table's holders come back with ensure_table's answer, and attach says them once per change rather than a boot's warning a try. Attach counts the role it runs as among its own; doctor does not. The dead SCIM recheck and the unused scim_ok parameter are gone. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 18 +- server/cli/cell_attach_beats.py | 4 +- server/cli/cell_attach_pg.py | 71 ++++-- server/cli/doctor_cell.py | 2 +- server/core/cell_triggers.py | 55 +++-- tests/integration/test_cell_attach.py | 71 +++++- .../integration/test_cell_org_triggers_pg.py | 6 +- tests/unit/test_cell_attach.py | 208 +++++++++++++++++- 8 files changed, 368 insertions(+), 67 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 52abea19c6..23449ccff9 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -282,6 +282,8 @@ async def stores_refusals(self) -> list[Refusal]: if recorded is None or str(recorded["deployment_id"]) != expected: out.append(Refusal("clickhouse", f"records another deployment id than {expected!r}")) state, first = await apg.read_mode(self.lock_conn) + if state in cell.MODES: + self.mode = state # what a refusal from here on leaves the cell at if first != (True, self.name): out.append( Refusal("postgres loom.cell_first", f"reads {first}, not (true, {self.name})") @@ -353,7 +355,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: async def catalog_refusals(self) -> list[Refusal]: """The Postgres checks that run again before the mode commit and in step 4.""" - holders = await apg.privilege_holders(self.lock_conn) + holders = await apg.privilege_holders(self.lock_conn, exempt_self=True) self.report["own_roles"] = holders.own out = apg.privilege_refusals(holders, self.args.pg_allow_role) out += await apg.unmarked_refusals(self.lock_conn, await apg.session_passes()) @@ -450,9 +452,9 @@ async def ready_for_commit(self) -> bool: async def commit_mode(self) -> None: from server.core import pg - # From here on the mode may have risen: only the durable read can say. - self.mode = "unknown" await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + # From the mode transaction on the mode may have risen: only the durable read can say. + self.mode = "unknown" try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( @@ -491,6 +493,8 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous after the durable read") break + if read.mode in cell.MODES: + self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": raise _refused( "postgres cell state", @@ -666,9 +670,7 @@ async def precommit(self, found: cell_scan.ScanResult) -> None: ) if out: raise Refused(out) - verdict = await self.check_beats("refuse") - if refusals := beats.final_refusals(verdict): - raise Refused(refusals) + await self.check_beats("refuse") async def dry_run(self, topology: Any, listed: cell_scan.TableList) -> int: found = await self.scan(topology, listed) @@ -711,6 +713,10 @@ def log(text: str) -> None: log(f"stopped ({type(e).__name__}: {e}); run it again") log(mode_line(run.mode)) return 1 + except BaseException: + # An interrupt: the mode line, then the interrupt as it came. + log(mode_line(run.mode)) + raise run.report["seconds"] = round(time.monotonic() - started, 1) print(json.dumps(run.report, indent=2, sort_keys=True, default=str)) return code diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index a9c6e80377..c2a8ed2428 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -180,7 +180,7 @@ def _stopped(out: Verdict, b: Beat, what: str, stopped_unset: StoppedUnset) -> N ) -def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: +def final_refusals(verdict: Verdict) -> list[Refusal]: out = [] if not verdict.live and not verdict.booting: out.append(Refusal("processes", "no live process of this cell reports a heartbeat")) @@ -192,7 +192,7 @@ def final_refusals(verdict: Verdict, *, scim_ok: bool = False) -> list[Refusal]: "finish the root-secret rollout first", ) ) - if verdict.scim and not scim_ok: + if verdict.scim: out.append( Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 076dc4a196..9c6e4708d6 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -257,15 +257,16 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: ORDER BY 1, 2 """ -# The roles attach counts as its own: superusers, and a login role that itself owns +# The roles attach counts as its own: superusers, a login role that itself owns # loom.tenancy, loom.tenancy_mode or a guarded table (it can drop or disable the -# triggers, or lower the mode, whatever else it holds). +# triggers, or lower the mode, whatever else it holds), and, for attach only, the role it +# runs as ($2): the operator runs attach, while doctor may run as a CI or monitoring role. _OWN_ROLES = """ SELECT rolname FROM pg_roles WHERE rolcanlogin AND (rolsuper OR oid IN ( SELECT c.relowner FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' AND c.relkind IN ('r', 'p') AND (c.relname IN ('tenancy', 'tenancy_mode') OR c.relname = ANY($1::text[])) -)) OR rolname = current_user +)) OR ($2 AND rolname = current_user) ORDER BY 1 """ @@ -279,9 +280,11 @@ class Holders: others: dict[str, list[str]] -async def privilege_holders(conn: Any) -> Holders: +async def privilege_holders(conn: Any, *, exempt_self: bool) -> Holders: + """The holders of a right `_RIGHTS` counts; `exempt_self` counts the role this session + runs as among the own roles (attach's, not doctor's).""" guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded)] + own = [str(r["rolname"]) for r in await conn.fetch(_OWN_ROLES, guarded, exempt_self)] others: dict[str, list[str]] = {} for r in await conn.fetch(_RIGHTS): if r["role"] not in own: @@ -449,42 +452,49 @@ async def _ensure_table( conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: yielded = False + said = "" while True: - lock_ms = 100 + most_ms = 100 holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): - lock_ms, yielded = deadlock_ms + 500, True - lock_ms = wait.bound_ms(lock_ms) - result = "busy" if lock_ms <= 0 else await _ensure_one(conn, table, lock_ms, wait) - if result == "applied": + most_ms, yielded = deadlock_ms + 500, True + outcome = await _ensure_one(conn, table, most_ms, wait) + if outcome.answer == "applied": return None - if result == "denied": + if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): return _busy_refusal(table, await cell_triggers.holders(conn, table)) - log(f"loom.{table}: its lock is busy; retrying") + # Said again only when the holders change: a retry a second is otherwise silent. + if outcome.holders != said: + said = outcome.holders + log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") -async def _ensure_one(conn: Any, table: str, lock_ms: int, wait: Wait) -> str: - """One try under the migration lock, whose own wait is bounded by what `wait` has left.""" +async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: + """One try under the migration lock: the migration lock's wait, and then the DDL's lock + wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" from server.core.schema.pg_migrate import _LOCK_KEY migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) if migration_ms <= 0: - return "busy" + return cell_triggers.Outcome("busy") await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return "busy" # a boot migrating: the next try + return cell_triggers.Outcome("busy") # a boot migrating: the next try raise finally: await conn.execute("RESET lock_timeout") try: + lock_ms = wait.bound_ms(most_ms) + if lock_ms <= 0: + return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 ) @@ -522,6 +532,9 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: """ _TIMED_OUT = frozenset({"55P03", "57014"}) +# How long attach waits for an ended mode transaction's lock to clear. +_END_ROUNDS = 30 # not a setting: checks of the lock +_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -585,10 +598,13 @@ async def _mode_try(conn: Any, lock_ms: int) -> bool: async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait until its lock - clears; Refused when its role is a superuser and attach's is not.""" - for _ in range(30): - rows = await conn.fetch(_MODE_HOLDERS) + """End an earlier run's mode transaction still in flight and wait, for up to + `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role + is a superuser and attach's is not, or when a backend still holds the lock after the + last check, naming it.""" + rows: list[Any] = [] + for _ in range(_END_ROUNDS): + rows = list(await conn.fetch(_MODE_HOLDERS)) if not rows: return superuser = bool( @@ -608,7 +624,20 @@ async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> N ) await confirm_lock() await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(0.1) + await _sleep(_END_PAUSE_S) + if not (rows := list(await conn.fetch(_MODE_HOLDERS))): + return + raise Refused( + [ + Refusal( + f"postgres backend {r['pid']}", + f"still holds loom.tenancy_mode's lock after it was ended " + f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " + "once it has gone", + ) + for r in rows + ] + ) # -- the durable read ------------------------------------------------------------------- diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 9edb8e275d..e8a5fc83c1 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -95,7 +95,7 @@ async def _writer_reads(allowed_roles: tuple[str, ...]) -> list[tuple[str, str, from server.core import pg async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=False) unchecked = await cell_attach_pg.unmarked_refusals(conn, False) roles = cell_attach_pg.privilege_refusals(holders, allowed_roles) return [ diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 4395138b4d..7a1ee5e5b7 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -21,7 +21,7 @@ import dataclasses import logging import re -from typing import Any, Literal +from typing import Any, Literal, NamedTuple from server.core import cell, pg, readiness @@ -95,6 +95,14 @@ # What a table's DDL came to (`ensure_table`, `_run_ddl`). Applied = Literal["applied", "busy", "denied"] + +class Outcome(NamedTuple): + """A table's DDL's answer, and with `busy` the sessions it waited behind.""" + + answer: Applied + holders: str = "" + + # pg_trigger.tgtype bits (src/include/catalog/pg_trigger.h). _ROW, _BEFORE, _INSERT, _UPDATE, _INSTEAD = 1, 2, 4, 16, 64 # The events each trigger must fire on. @@ -265,14 +273,15 @@ async def ensure_table( recording_first: bool = False, lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: +) -> Outcome: """`ensure` for one table of the planned set (`loom.tenancy` its guard, any other its org triggers), so one table's DDL can run alone: in `enforced`, or with `recording_first` also in a first cell's `recording`, where the org triggers' function passes every row and a boot never creates them. `applied` when the DDL ran or nothing was needed, `busy` when its lock was not taken in `attempts` tries of `lock_timeout`, - `denied` when this role cannot alter the table. Raises for a table outside the planned - set, or in another state.""" + `denied` when this role cannot alter the table; a `busy` answer carries the sessions it + waited behind and logs nothing, so the caller says it as often as it retries. Raises + for a table outside the planned set, or in another state.""" state = await cell.read_state(conn) if state != "enforced" and not (recording_first and state == "recording"): raise RuntimeError(f"cell triggers: not created in state {state!r}") @@ -284,8 +293,8 @@ async def ensure_table( ((_, names),) = planned sqls = _table_statements(table, names, await _triggers(conn, names)) if not sqls: - return "applied" - return await _run_ddl(conn, table, sqls, lock_timeout, attempts) + return Outcome("applied") + return await _run_ddl(conn, table, sqls, lock_timeout, attempts, warn=False) def _table_statements( @@ -308,11 +317,14 @@ async def _run_ddl( sqls: list[str], lock_timeout: str = LOCK_TIMEOUT, attempts: int = LOCK_ATTEMPTS, -) -> Applied: + *, + warn: bool = True, +) -> Outcome: """A table's DDL in one transaction, `attempts` short tries: `applied`, or which of the - two causes cells.md names kept it from applying (`busy`: its lock not taken; `denied`: - a table this role cannot alter), which a boot serves past; any other error fails it, as - a failed read of the state does.""" + two causes cells.md names kept it from applying (`busy`: its lock not taken, with the + sessions it waited behind; `denied`: a table this role cannot alter), which a boot + serves past; any other error fails it, as a failed read of the state does. `warn` logs + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" import asyncpg for attempt in range(attempts): @@ -321,25 +333,28 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - return "applied" + return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) - return "denied" + return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) continue - log.warning( - "cell triggers: not applied on loom.%s (%s): %s; %s", - table, - e, - "; ".join(sqls), - await _holders(conn, table), - ) - return "busy" + held = await _holders(conn, table) + if warn: + log.warning( + "cell triggers: not applied on loom.%s (%s): %s; %s", + table, + e, + "; ".join(sqls), + held, + ) + return Outcome("busy", held) + return Outcome("busy") async def holders(conn: Any, table: str) -> list[Any]: diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index e0f96f409f..d77592d1dc 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,7 +532,7 @@ async def test_each_right_that_makes_an_org_held_is_named( await conn.execute("GRANT USAGE ON SCHEMA loom TO " + role) for sql in grant.format(r=role, owner=owner).split("; "): await conn.execute(sql) - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) assert owner in holders.own assert any(why in w for w in holders.others.get(role, [])), holders.others finally: @@ -550,7 +550,7 @@ async def test_a_function_owner_counts_though_it_owns_no_table(cell_stores) -> N try: async with pg.dedicated() as conn: await conn.execute(f"ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO {role}") - holders = await cell_attach_pg.privilege_holders(conn) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) await conn.execute("ALTER FUNCTION loom.tenancy_refuse_unheld() OWNER TO CURRENT_USER") assert holders.others[role] == ["owns loom.tenancy_refuse_unheld()"] finally: @@ -631,8 +631,40 @@ async def test_doctor_names_stray_orgs_smallest_tables_first(cell_stores) -> Non lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 300)} grade, detail = lines["cell_stray_orgs"] assert grade == "WARN" and tag + "stray" in detail and tag + "held" not in detail - tight = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 0.001)} - assert tight["cell_stray_orgs_unchecked"][0] == "WARN" + + +@pytest.mark.asyncio +async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( + cell_stores, monkeypatch +) -> None: + from server.cli import cell_scan, doctor_cell + + stores = cell_stores + await _prepare(stores, _Network()) + tag = stores["tag"] + await pg.execute("CREATE TABLE loom.it_small (org text PRIMARY KEY)") + await pg.execute("CREATE TABLE loom.it_large (org text PRIMARY KEY)") + await pg.execute("INSERT INTO loom.it_small VALUES ($1)", tag + "small") + await pg.execute( + "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" + ) + await pg.execute("ANALYZE loom.it_small, loom.it_large") + # The budget runs out on the largest table, as it does on a real cell when that table's + # read is what is too slow: every smaller one was read first. + real, read = cell_scan.pg_orgs, [] + + async def pg_orgs(table: str, budget: Any) -> list[str]: + read.append(table) + if table == "loom.it_large": + await asyncio.sleep(budget.left() + 0.1) + return await real(table, budget) + + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} + assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert tag + "small" in lines["cell_stray_orgs"][1] + grade, detail = lines["cell_stray_orgs_unchecked"] + assert grade == "WARN" and detail.endswith(": loom.it_large") @pytest.mark.asyncio @@ -657,10 +689,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. - assert ( - await cell_triggers.ensure_table(conn, "scope_generations", recording_first=True) - == "denied" + outcome = await cell_triggers.ensure_table( + conn, "scope_generations", recording_first=True ) + assert outcome.answer == "denied" await conn.execute("RESET ROLE") assert await _mode() == "recording" finally: @@ -688,3 +720,28 @@ async def test_doctor_names_a_table_it_cannot_read_and_reads_the_rest(cell_store grade, detail = lines["cell_stray_orgs_unread"] assert grade == "WARN" and "postgres loom.it_no_org_index" in detail assert "cell_stray_orgs_unchecked" not in lines + + +@pytest.mark.asyncio +async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_stores) -> None: + await _prepare(cell_stores, _Network()) + role = f"it_ci_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA loom TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"SET ROLE {role}") + as_attach = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + as_doctor = await cell_attach_pg.privilege_holders(conn, exempt_self=False) + await conn.execute("RESET ROLE") + # Attach runs as the operator's role; a CI role doctor runs as is warned about. + assert role in as_attach.own and role not in as_attach.others + assert role not in as_doctor.own + assert "may insert into loom.tenancy" in as_doctor.others[role] + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index b2fb66a467..c219d05777 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -722,15 +722,15 @@ async def test_ensure_table_creates_a_recording_first_cells_triggers_only_when_a with pytest.raises(RuntimeError, match="recording"): await cell_triggers.ensure_table(conn, "scope_generations") ensure = cell_triggers.ensure_table - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" - assert await ensure(conn, "tenancy", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" + assert (await ensure(conn, "tenancy", recording_first=True)).answer == "applied" assert await _cell_triggers(conn) == { ("scope_generations", "tenancy_insert", "O"), ("scope_generations", "tenancy_update", "O"), ("tenancy", "tenancy_record", "O"), } # Again: nothing to do. - assert await ensure(conn, "scope_generations", recording_first=True) == "applied" + assert (await ensure(conn, "scope_generations", recording_first=True)).answer == "applied" with pytest.raises(ValueError, match="no cell triggers"): await cell_triggers.ensure_table(conn, "idempotency", recording_first=True) plain = await _unmarked(dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 857d12dfe0..b3db51ee02 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -268,8 +268,8 @@ async def planned(_c: Any) -> list[str]: async def holders(_c: Any, _t: str) -> list[Any]: return [] - async def ensure_table(*_a: Any, **_kw: Any) -> str: - return "denied" + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("denied") monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", holders) @@ -280,14 +280,22 @@ async def ensure_table(*_a: Any, **_kw: Any) -> str: assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 - async def busy(*_a: Any, **_kw: Any) -> str: + holders = iter(["pid 7", "pid 7", "pid 7", "pid 8"] + ["pid 8"] * 100) + + async def busy(*_a: Any, **_kw: Any) -> Any: clock.now += 0.1 - return "busy" + return cell_triggers.Outcome("busy", next(holders)) monkeypatch.setattr(cell_triggers, "ensure_table", busy) - wait = apg.Wait(3, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) - assert "not taken within --wait" in refusal.reason and wait.spent <= 3 + wait = apg.Wait(10, clock) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # Said once per change of holders, not once a try. + assert logged == [ + "loom.scope_generations: its lock is busy (pid 7); retrying", + "loom.scope_generations: its lock is busy (pid 8); retrying", + ] # -- the rescan's watch ---------------------------------------------------------------- @@ -828,3 +836,189 @@ async def unread(_roles: Any) -> Any: lines = {n: (g, d) for n, g, d in await doctor_cell._checks(None)} for name in ("cell_tenancy_privilege", "cell_unchecked_writers"): assert lines[name] == ("WARN", "not read: OSError: the catalog read failed") + + +@pytest.mark.asyncio +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: + """A migration lock that took the rest of --wait leaves no DDL try.""" + + def migration_lock(*_a: Any) -> str: + clock.now += 2.0 # the rest of --wait + return "OK" + + conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) + wait = apg.Wait(2, clock) + with wait.counting(): + outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) + assert outcome.answer == "busy" + assert not any( + "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + ) + + +@pytest.mark.asyncio +async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: + held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] + conn = FakeConn( + [ + ("l.mode = 'ExclusiveLock'", held), + ("rolsuper FROM pg_roles", False), + ("pg_terminate_backend", True), + ] + ) + + async def confirm() -> None: + pass + + with pytest.raises(Refused) as e: + await apg.end_mode_transactions(conn, confirm) + (refusal,) = e.value.refusals + assert refusal.subject == "postgres backend 5" + assert "still holds loom.tenancy_mode's lock" in refusal.reason + + +def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield None + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Run, "run", body) + + +@pytest.mark.asyncio +async def test_a_router_refusal_of_attach_enforced_leaves_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: + raise Refused([Refusal("router", f"refused {path}")]) + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + await self.commit_mode() + return 0 + + monkeypatch.setattr(cell_attach.Router, "call", refuse) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_a_durable_read_that_sees_recording_says_recording( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def synchronous() -> str: + return "synchronous" + + async def read() -> apg.Durable: + return apg.Durable(mode="recording", first=(True, "first"), xid=1) + + async def run(self: Any) -> int: + self.mode = "unknown" # a mode commit whose outcome was lost + await self.durable() + return 0 + + monkeypatch.setattr(pg_profile, "standby_now", synchronous) + monkeypatch.setattr(apg, "durable_read", read) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + + +@pytest.mark.asyncio +async def test_an_enforced_resume_refused_early_says_enforced( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + from server.core import pg_profile + + async def recorded(_ch: Any) -> dict[str, Any]: + return {"deployment_id": "other"} + + async def health(_self: Any, _host: str) -> str: + return "dep-1" + + async def read_mode(_conn: Any) -> tuple[str, Any]: + return "enforced", (True, "first") + + async def run(self: Any) -> int: + self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} + self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) + if refusals := await self.stores_refusals(): + raise Refused(refusals) + return 0 + + monkeypatch.setattr(pg_profile, "recorded_authority", recorded) + monkeypatch.setattr(cell_attach.Router, "health", health) + monkeypatch.setattr(apg, "read_mode", read_mode) + _stub_attach(monkeypatch, run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + err = capsys.readouterr().err.splitlines() + assert "clickhouse: records another deployment id" in err[0] + assert err[-1] == ( + "loom cell attach: the cell stays enforced and attaching: rerun attach to resume" + ) + + +@pytest.mark.asyncio +async def test_an_interrupt_prints_the_mode_line_and_goes_on( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + async def run(self: Any) -> int: + self.mode = "unknown" + raise KeyboardInterrupt + + _stub_attach(monkeypatch, run) + with pytest.raises(KeyboardInterrupt): + await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) + assert "the mode commit's outcome is unknown" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_stray_read_names_a_refused_table_and_reads_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.cli import cell_ch, doctor_cell + from server.core import pg + + async def topology(_ch: Any) -> Any: + return cell_ch.Topology("", 0) + + async def no_tables(*_a: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + async def sizes(*_a: Any) -> dict[str, int]: + return {} + + pg_rows = [{"name": "loom.a_unindexed", "n": 1}, {"name": "loom.b_data", "n": 2}] + conn = FakeConn([("FROM loom.tenancy", []), ("pg_class c", pg_rows)]) + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + async def pg_orgs(table: str, _budget: Any) -> list[str]: + if table == "loom.a_unindexed": + raise Refused([Refusal(f"postgres {table}", "has no plain btree index led by org")]) + return ["stray-org"] + + monkeypatch.setattr(cell_ch, "read_topology", topology) + monkeypatch.setattr(cell_scan, "list_clickhouse", no_tables) + monkeypatch.setattr(doctor_cell, "_table_sizes", sizes) + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) + lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(None, 60)} + assert lines["cell_stray_orgs"] == ("WARN", "postgres loom.b_data: stray-org") + assert lines["cell_stray_orgs_unread"] == ( + "WARN", + "not read: postgres loom.a_unindexed: has no plain btree index led by org", + ) + assert "cell_stray_orgs_unchecked" not in lines From 4a25c41ba6924963b6352e8fca17272cfae4abf5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:37:50 +0000 Subject: [PATCH 026/181] The attach docs say which role attach exempts, and the trigger gate's figures say what they establish cells.md and the runbook state that attach exempts the role it runs as and doctor does not. results.md gives the marked batch's per-round changes and their mean, says the delivery-pass gate is met on the mean but not established by three rounds alone, and that the memory.set p99 gate is unresolved locally. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +++++---- docs/benchmarks/results.md | 19 +++++++++++-------- docs/operations/cell-attach.md | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 3ea7ddab80..246da73b77 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1007,10 +1007,11 @@ It refuses, before any write, naming each: without one, or a role that may not write the durable read's logical message; a role of attach's own lacking its grants (`pg_read_all_stats` among them) or a server with `max_prepared_transactions` above 0; -- **the tenancy-privilege check**: a login role, other than attach's own (superusers, - and a role that itself owns `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and - those named with `--pg-allow-role`, that holds or reaches through any membership a right - to make an org held or lower the mode: `INSERT` on any column of `loom.tenancy` or +- **the tenancy-privilege check**: a login role, other than attach's own (the role attach + runs as, which the operator runs it with; superusers; and a role that itself owns + `loom.tenancy`, `loom.tenancy_mode` or a guarded table) and those named with + `--pg-allow-role`, that holds or reaches through any membership a right to make an org + held or lower the mode: `INSERT` on any column of `loom.tenancy` or `UPDATE` of its `org`, `INSERT` on `loom.tenancy_mode` or `UPDATE` of its `mode`, `TRIGGER` on a `loom` table, `CREATE` on the database or a schema that is not a system one (a function there can shadow one the servers call unqualified on their marked diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 81e099d3c8..4814bd7f5f 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1241,14 +1241,17 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch, which is what a delivery pass of a cell's own processes writes, shows no -cost the rounds can tell from zero (on minus off: +1.3, −1.9, −2.1 ms), within the 10% gate. -The `memory.set` figures move by more between rounds of one configuration than any trigger -cost: on this loaded host the p99 differences within a round (+214.8, +47.0, +24.3 ms -here, and −71.7, −10.0, −42.4 ms in an earlier run of the same measurement) have no -consistent sign, so this run cannot resolve a 10 ms p99 gate; it shows no rise beyond the -host's noise, and the statement-level figures above bound the marked cost at under a -microsecond a row. An unmarked session pays about 11 µs a row, as above. +The marked batch is what a delivery pass of a cell's own processes writes. On minus off +within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are +9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is +met on the mean, but one round of three was 13% slower and three rounds give no paired bound +(methodology's paired bootstrap is for per-item labels), so this run does not establish the +gate on its own; the statement-level figures above do, at under a microsecond a row against +about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one +configuration than any trigger cost: on this loaded host the p99 differences within a round +(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same +measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is +unresolved locally. An unmarked session pays about 11 µs a row, as above. ## Reference computations diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index c51541d06d..481b9a95ef 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -135,10 +135,11 @@ procedure as an over-long org's. `loom doctor` runs attach's point-in-time checks again on every run: it warns on a Postgres role outside the owner roles, the superusers and its own `--pg-allow-role` that can make an -org held or lower the mode (`cell_tenancy_privilege`), and on a role or database setting or -a subscription that passes an unchecked writer (`cell_unchecked_writers`). A grant or a -writer added after `serving` passes attach's checks, which ran once; doctor is how it is -found. +org held or lower the mode (`cell_tenancy_privilege`). Attach exempts the role it runs as, +the operator's; doctor does not exempt the role it runs as, so a CI or monitoring role +holding such a right is named. It warns too on a role or database setting or a subscription +that passes an unchecked writer (`cell_unchecked_writers`). A grant or a writer added after +`serving` passes attach's checks, which ran once; doctor is how it is found. ## Related From 0c37c6860b08744f4bbf9afdf2b0d45681e83855 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 027/181] Every attach try runs once whatever --wait has left, the mode reads unknown only while a try's UPDATE is out, and a busy migration lock is named Each trigger and mode-commit try gets a lock wait of at least TRY_FLOOR_MS before --wait is judged spent, so --wait 0 is one try of each wait with no retry, and an idle cell attaches with it. The run's mode turns unknown just before a try sends its UPDATE and back to recording when that try rolls back. A trigger try that finds the migration lock busy names that lock's holder, in the retry line and in the refusal. The step-4 watch task is awaited after its cancel. The ATTACH_LOCK_KEY comment says who holds the lock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 24 +++- server/cli/cell_attach_pg.py | 74 ++++++++--- server/core/cell.py | 6 +- tests/integration/test_cell_attach.py | 17 ++- tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_attach.py | 176 ++++++++++++++++++++++++-- 7 files changed, 260 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 23449ccff9..8ed70e2d37 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -30,6 +30,7 @@ import argparse import asyncio +import contextlib import json import sys import time @@ -453,16 +454,22 @@ async def commit_mode(self) -> None: from server.core import pg await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - # From the mode transaction on the mode may have risen: only the durable read can say. - self.mode = "unknown" + + def mark(mode: str) -> None: + # `unknown` from a try's UPDATE on: only the durable read can say whether it + # landed; `recording` again when that try rolled back. + self.mode = mode + try: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark ) except Refused: raise - except Exception as e: # noqa: BLE001 — the durable read says whether it landed + except Exception as e: + if self.mode != "unknown": + raise # no try was in flight: nothing to decide self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -573,6 +580,8 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: raise Refused(writers) finally: task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp @@ -757,7 +766,12 @@ async def _run(ch: Any) -> int: def add_attach_parser(csub: Any) -> None: p = csub.add_parser("attach", help="Attach the prepared first cell: place its orgs, serve.") p.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") - p.add_argument("--wait", type=int, default=_WAIT_S, help="Seconds the waits share") + p.add_argument( + "--wait", + type=int, + default=_WAIT_S, + help="Seconds of wall time the waits share; 0 runs one try of each, with no retry", + ) p.add_argument( "--scan-budget", type=int, default=_SCAN_BUDGET_S, help="Seconds one full org scan may take" ) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 9c6e4708d6..0ca9fefcc1 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -24,6 +24,8 @@ # The pause between polls of a wait. POLL_S = 1.0 # not a setting: a poll interval +# The least lock wait a try gets, however little of --wait is left. +TRY_FLOOR_MS = 100 # not a setting: one try's floor # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound @@ -67,8 +69,10 @@ def counting(self) -> Iterator[None]: self._since = None def bound_ms(self, most_ms: int) -> int: - """A lock timeout for one try: `most_ms`, or what is left when that is less.""" - return min(most_ms, int(self.left() * 1000)) + """A lock timeout for one try: `most_ms`, or what is left when that is less, and + never under `TRY_FLOOR_MS`: every try runs once, so `--wait 0` is one try of each + wait with no retry, and only after a try is the wait judged spent.""" + return max(TRY_FLOOR_MS, min(most_ms, int(self.left() * 1000))) async def pause(self, seconds: float = POLL_S) -> bool: """Wait `seconds` (at most what is left); False once none is left.""" @@ -89,6 +93,13 @@ def _split(key: int) -> str: ) +# Who holds an advisory lock (`where` from `_split`), never a query's text. +_ADVISORY_HOLDER = """ +SELECT l.pid, a.application_name, a.backend_start +FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid +WHERE {where} AND l.granted +""" + _LOCK_HOLDER = f""" SELECT l.pid, a.application_name, a.backend_start, a.client_addr::text AS client FROM pg_locks l LEFT JOIN pg_stat_activity a ON a.pid = l.pid @@ -467,6 +478,11 @@ async def _ensure_table( "attach's role cannot alter it: make the role a member of its owner role", ) if not await wait.pause(): + if outcome.holders.startswith(_MIGRATION_HELD): + return Refusal( + f"postgres loom.{table}", + f"its triggers were not created within --wait: {outcome.holders}", + ) return _busy_refusal(table, await cell_triggers.holders(conn, table)) # Said again only when the holders change: a retry a second is otherwise silent. if outcome.holders != said: @@ -474,29 +490,45 @@ async def _ensure_table( log(f"loom.{table}: its lock is busy ({said or 'no holder named'}); retrying") +_MIGRATION_HELD = "the migration lock" + + +async def _migration_holder(conn: Any, key: int) -> str: + """`the migration lock (held by …)`, naming the session that holds it.""" + try: + rows = await conn.fetch(_ADVISORY_HOLDER.format(where=_split(key))) + except Exception as e: # noqa: BLE001 — naming the holder is best effort + return f"{_MIGRATION_HELD} (its holder not read: {type(e).__name__})" + held = "; ".join( + f"pid {r['pid']}, {r['application_name'] or 'no name'}, since {r['backend_start']}" + for r in rows + ) + return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" + + async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock - wait (at most `most_ms`), each bounded by what `wait` has left when it starts.""" + wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the + try floor. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - migration_ms = wait.bound_ms(_MIGRATION_LOCK_MS) - if migration_ms <= 0: - return cell_triggers.Outcome("busy") - await conn.execute(f"SET lock_timeout = '{migration_ms}ms'") + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) except Exception as e: if getattr(e, "sqlstate", None) == "55P03": - return cell_triggers.Outcome("busy") # a boot migrating: the next try + # A boot migrating: the next try. + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) raise finally: await conn.execute("RESET lock_timeout") try: - lock_ms = wait.bound_ms(most_ms) - if lock_ms <= 0: - return cell_triggers.Outcome("busy") return await cell_triggers.ensure_table( - conn, table, recording_first=True, lock_timeout=f"{lock_ms}ms", attempts=1 + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, ) finally: await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) @@ -543,21 +575,29 @@ async def commit_enforced( ready: Callable[[], Any], wait: Wait, log: Callable[[str], None], + mark: Callable[[str], None], ) -> None: """Raise the mode to `enforced` on `conn` (a session of its own, never the lock session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before each try (False skips the try while a process boots). After a failed try the in-flight - mode transaction, found by its lock, is ended. Raises Refused when `wait` runs out; - whether the commit landed is the durable read's to say.""" + mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try + sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` + runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - lock_ms = wait.bound_ms(_MODE_LOCK_MS) - if lock_ms > 0 and await ready(): - if await _mode_try(conn, lock_ms): + if await ready(): + mark("unknown") + try: + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + except Refused: + mark("recording") # the UPDATE's transaction rolled back + raise + if done: return + mark("recording") log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..37e20f99cb 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -62,9 +62,9 @@ # Postgres's btree entry limit (about 2.7 KB), so every org a cell records has a # placement the directory can index. ORG_MAX_BYTES = 2048 # not a setting: cells and the router judge one limit -# The advisory lock on a cell's Postgres that attach will hold, and prepare and update once -# they take it, and that the troubleshooting page's full-state cleanup takes before the -# migration lock. Positive: the page's holder query splits it into two `oid` halves. +# The advisory lock on a cell's Postgres that attach and prepare hold for their run, and that +# the troubleshooting page's full-state cleanup takes before the migration lock. Positive: +# the page's holder query splits it into two `oid` halves. ATTACH_LOCK_KEY = 0x6C6F6F6D_61746368 # "loomatch" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index d77592d1dc..65064ab365 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -482,7 +482,9 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.commit_enforced(conn, ok, ok, cell_attach_pg.Wait(20), logged.append) + await cell_attach_pg.commit_enforced( + conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + ) await task assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] @@ -745,3 +747,16 @@ async def test_attach_exempts_the_role_it_runs_as_and_doctor_does_not(cell_store await conn.execute(f"DROP OWNED BY {role}") await _admin(f"DROP OWNED BY {role}") await _admin(f"DROP ROLE {role}") + + +@pytest.mark.asyncio +async def test_wait_0_attaches_an_idle_cell(cell_stores, capsys) -> None: + """--wait 0 is one try of each wait, no retry: an idle cell still reaches serving.""" + stores, network = cell_stores, _Network() + await _prepare(stores, network) + org = stores["tag"] + "w0" + await pg.execute("INSERT INTO loom.scope_generations (org, scope) VALUES ($1, '*')", org) + capsys.readouterr() + assert await _attach(stores, network, wait=0) == 0, capsys.readouterr().err + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert org in await _placed() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 3e88dbafca..4e45db4285 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -461,7 +461,7 @@ ], "loom cell attach": [ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds the waits share --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 3d597c1e03..a31a862106 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13032,7 +13032,7 @@ "nargs": null, "type": "builtins:int", "required": false, - "help": "Seconds the waits share", + "help": "Seconds of wall time the waits share; 0 runs one try of each, with no retry", "metavar": null }, { diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index b3db51ee02..429c2fcfb7 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -250,7 +250,7 @@ async def ready() -> bool: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None) + await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -290,7 +290,9 @@ async def busy(*_a: Any, **_kw: Any) -> Any: wait = apg.Wait(10, clock) logged: list[str] = [] (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) - assert "not taken within --wait" in refusal.reason and wait.spent <= 10 + # The last try ran with the floor's lock wait, after --wait was all but spent. + assert "not taken within --wait" in refusal.reason + assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 # Said once per change of holders, not once a try. assert logged == [ "loom.scope_generations: its lock is busy (pid 7); retrying", @@ -654,7 +656,9 @@ async def ready() -> bool: return True logged: list[str] = [] - await apg.commit_enforced(conn, confirm, ready, apg.Wait(10, clock), logged.append) + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + ) assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 @@ -692,7 +696,9 @@ async def booting() -> bool: return False with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, confirm, booting, apg.Wait(2, clock), lambda _t: None) + await apg.commit_enforced( + conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + ) assert not any("LOCK TABLE" in s for s in conn.executed) @@ -789,8 +795,8 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*_a: Any) -> None: - pass # the commit landed; its answer may not have + async def landed(*a: Any) -> None: + a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} @@ -839,22 +845,166 @@ async def unread(_roles: Any) -> Any: @pytest.mark.asyncio -async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock(clock: _Clock) -> None: - """A migration lock that took the rest of --wait leaves no DDL try.""" +async def test_the_ddl_lock_bound_is_taken_after_the_migration_lock( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + """A migration lock that took the rest of --wait leaves the DDL its floor: one try, never + the full 100 ms when less is left, never none.""" + from server.core import cell_triggers def migration_lock(*_a: Any) -> str: - clock.now += 2.0 # the rest of --wait + clock.now += 1.95 # all but 50 ms of --wait return "OK" + asked: list[str] = [] + + async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: + asked.append(lock_timeout) + return cell_triggers.Outcome("applied") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 100, wait) - assert outcome.answer == "busy" - assert not any( - "lock_timeout = '" in s and "ms'" in s and "SET LOCAL" in s for s in conn.executed + outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + assert outcome.answer == "applied" + assert asked == [f"{apg.TRY_FLOOR_MS}ms"] + + +@pytest.mark.asyncio +async def test_a_busy_migration_lock_is_named_as_such( + monkeypatch: pytest.MonkeyPatch, clock: _Clock +) -> None: + from server.core import cell_triggers + + class _Busy(Exception): + sqlstate = "55P03" + + holder = [{"pid": 42, "application_name": "loom", "backend_start": "t0"}] + conn = FakeConn( + [ + ("pg_advisory_lock", _Busy()), + ("l.locktype = 'advisory'", holder), + ("deadlock_timeout", 1000), + ] ) + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + logged: list[str] = [] + (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + named = "the migration lock (held by pid 42, loom, since t0)" + assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] + assert refusal.reason == f"its triggers were not created within --wait: {named}" + + +@pytest.mark.asyncio +async def test_wait_0_runs_one_try_of_each_wait( + clock: _Clock, monkeypatch: pytest.MonkeyPatch +) -> None: + """On an idle database --wait 0 commits the mode and creates the triggers in one try + each, and the heartbeats pass when nothing boots.""" + from server.core import cell_triggers + + conn = FakeConn( + [ + ("LOCK TABLE", "LOCK TABLE"), + ("UPDATE loom.tenancy_mode", "UPDATE 1"), + ("l.mode = 'ExclusiveLock'", []), + ] + ) + marks: list[str] = [] + + async def ok() -> bool: + return True + + async def nothing() -> None: + pass + + await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + assert marks == ["unknown"] + assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) + + async def planned(_c: Any) -> list[str]: + return ["scope_generations"] + + async def no_holders(_c: Any, _t: str) -> list[Any]: + return [] + + async def applied(*_a: Any, **_kw: Any) -> Any: + return cell_triggers.Outcome("applied") + + async def no_defects(_c: Any) -> list[Any]: + return [] + + monkeypatch.setattr(cell_triggers, "planned_tables", planned) + monkeypatch.setattr(cell_triggers, "holders", no_holders) + monkeypatch.setattr(cell_triggers, "ensure_table", applied) + monkeypatch.setattr(cell_triggers, "missing", no_defects) + ddl = FakeConn([("deadlock_timeout", 1000)]) + assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + + async def read() -> list[beats.Beat]: + return [_beat("a")] + + assert (await beats.settle(read, _judge, apg.Wait(0, clock))).live == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refused_before_any_try", [True, False]) +async def test_a_mode_commit_refused_without_a_landed_try_says_recording( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + clock: _Clock, + refused_before_any_try: bool, +) -> None: + """The closing line says recording when no try sent its UPDATE (the heartbeats refused + before one) and when every try timed out and rolled back.""" + + def lock(*_a: Any) -> str: + raise _Timeout() + + conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + + async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: + return {"cell": "first"} + + async def ready(_self: Any) -> bool: + if refused_before_any_try: + raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) + return True + + async def confirm(_self: Any) -> None: + pass + + async def run(self: Any) -> int: + self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.wait = apg.Wait(2, clock) + await self.commit_mode() + return 0 + + from contextlib import asynccontextmanager + + from server.core import pg + + @asynccontextmanager + async def dedicated(**_kw: Any) -> Any: + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) + monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) + monkeypatch.setattr(cell_attach.Run, "run", run) + assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 + assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" + @pytest.mark.asyncio async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: From 07f9ea19618558e8b7307402720ae6f73a8fd846 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 08:51:15 +0000 Subject: [PATCH 028/181] The attach docs say what --wait 0 does, and cells.md keeps each code span on one line inside its list items Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/cells.md | 185 +++++++++++++++++---------------- docs/operations/cell-attach.md | 27 ++--- 2 files changed, 108 insertions(+), 104 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 246da73b77..53495c7a1d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1099,12 +1099,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): `enforced`; then `attach/serving`. `--wait` (default 10 minutes) is the wall time the bounded waits share, each lock try -included and never longer than what is left; `--scan-budget` (default 2 hours) bounds each -full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints -what would be placed and skipped and the scan's time, and writes nothing; the router's -checks at `attach/begin` do not run. On a `serving` row attach reads the cell's mode: -`enforced` is done (exit 0), and any other mode refuses: the directory and the cell's -Postgres disagree, as after a restore of one of them, and attach lowers and repairs neither. +included and never longer than what is left, but for a floor of 100 ms: every try runs once +before the wait is judged spent, so `--wait 0` is one try of each wait with no retry; +`--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. +`--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the +scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a +`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode +refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, +and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the @@ -1487,58 +1489,58 @@ An integration test with two cells and a router in CI's Postgres job: between prepare and attach, keeps serving on a first cell in `recording`, and keeps the last mode it read through a failed read, `recording` included; a CLI command declaring an org with it unset is refused on an `enforced` or `partial` cell, an org-free one (a token - revocation) runs, also with Postgres down on a deployment never prepared, and a `loom -cell` store command refuses; `loom cell limit --closed` succeeds with the cell's Postgres - down; after `loom cell credential --cell --next` both credentials place and read - state, `--finish` refuses the old credential, for a cell finished before its pods hold the - new one the router refuses its calls with 401, its request answers 503 writing nothing, - and it recovers by recording the old hash again, `--revert` drops a pending `next`, which - is then refused, and the old credential keeps placing, and a second `--next` while one is - pending and a hash another cell records as current or next are refused; `loom cell limit` - and `loom cell credential --next`, `--finish` and `--revert` refuse while the directory is - unavailable; concurrent placements at one under `placements_max` across two router - replicas write one; a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` set - refuses, naming it, on a first cell once it is `enforced` (`attaching` included) and on a - later cell, and on a first cell in `recording` runs and records the org; attach with - `placements_max` below the orgs the first cell holds places them all, and a later cell's - attach past its quota or closed refuses, naming the org; a first cell with `--closed` and - `placements_max` set before step 2 attaches, and an org a request records between step 2 - and `enforced` is placed by step 4 with no 503; after step 3's write of the cell's mode, a - process whose cache still reads `recording` sends a new org's request to the placement - call, which a closed cell answers 503; a CLI command whose insert finds `enforced` - refuses, naming the org, and makes no placement call; a refusal after step 3's write - leaves the cell `enforced` and `attaching`, where a new org is placed through the gate; - each ordering of attach's write of the mode with a recording insert holds as stated above - (running, starting during the write, a stream of overlapping inserts each shorter than a - try, with attach setting `enforced` within a try or two, held across several tries, held - past `--wait` (attach refuses, the cell still `recording`), a stalled attach, a commit cut - off by its timeout, and a commit whose connection is lost while it waits for the standby, - which the durable read decides); a `REPEATABLE READ` transaction whose first statement ran - before `enforced` committed, on attach's role or a second role, holds step 4 until it ends - and its write of an unheld org is then placed, one whose first statement ran after is - refused by the trigger, a transaction that starts after attach takes the lock but before - its commit is visible holds step 4, a transaction opened in `recording` that outlives one - run's `--wait` holds the rerun's step 4 after a refusal, an autovacuum worker on a loom - table does not hold step 4 and a background worker with a transaction open since - `recording` does, a prepared transaction (after a restart with `max_prepared_transactions` - raised during the wait) makes step 4 refuse at once, naming it, with the mode still - `enforced` and a rerun refused until it is resolved and the setting is back to 0, a write - transaction open in a second database on the same server holds step 4 and is named, a long - read on the synchronous standby, or on a second standby, with `hot_standby_feedback` on - does not hold it, nor does a base backup running across it, a write held open on a - `replication=database` connection holds step 4, and so does a `REPEATABLE READ` - transaction on one, opened in `recording` and writing nothing before `enforced`, a long - logical dump (a `REPEATABLE READ` read-only transaction) on the cell's database past - `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and the first - cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` above 0 - refuses before step 1 with the mode still `recording`, while a later cell's attach does - not require either; a missing or disabled org trigger is refused by attach and named by - `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was deleted - is refused and the state named by `loom doctor`; on that cell a pod with `LOOM_CELL` unset - is unready and runs no worker, and a CLI command declaring an org with it unset is - refused, as a pod is; updating, deleting or truncating the `first` row on an `enforced` - cell is refused, also under `session_replication_role = replica`, and a missing or - disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running + revocation) runs, also with Postgres down on a deployment never prepared, and a + `loom cell` store command refuses; `loom cell limit --closed` succeeds with the cell's + Postgres down; after `loom cell credential --cell --next` both credentials place + and read state, `--finish` refuses the old credential, for a cell finished before its pods + hold the new one the router refuses its calls with 401, its request answers 503 writing + nothing, and it recovers by recording the old hash again, `--revert` drops a pending + `next`, which is then refused, and the old credential keeps placing, and a second `--next` + while one is pending and a hash another cell records as current or next are refused; + `loom cell limit` and `loom cell credential --next`, `--finish` and `--revert` refuse + while the directory is unavailable; concurrent placements at one under `placements_max` + across two router replicas write one; a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, on a first cell once it is `enforced` + (`attaching` included) and on a later cell, and on a first cell in `recording` runs and + records the org; attach with `placements_max` below the orgs the first cell holds places + them all, and a later cell's attach past its quota or closed refuses, naming the org; a + first cell with `--closed` and `placements_max` set before step 2 attaches, and an org a + request records between step 2 and `enforced` is placed by step 4 with no 503; after step + 3's write of the cell's mode, a process whose cache still reads `recording` sends a new + org's request to the placement call, which a closed cell answers 503; a CLI command whose + insert finds `enforced` refuses, naming the org, and makes no placement call; a refusal + after step 3's write leaves the cell `enforced` and `attaching`, where a new org is placed + through the gate; each ordering of attach's write of the mode with a recording insert + holds as stated above (running, starting during the write, a stream of overlapping inserts + each shorter than a try, with attach setting `enforced` within a try or two, held across + several tries, held past `--wait` (attach refuses, the cell still `recording`), a stalled + attach, a commit cut off by its timeout, and a commit whose connection is lost while it + waits for the standby, which the durable read decides); a `REPEATABLE READ` transaction + whose first statement ran before `enforced` committed, on attach's role or a second role, + holds step 4 until it ends and its write of an unheld org is then placed, one whose first + statement ran after is refused by the trigger, a transaction that starts after attach + takes the lock but before its commit is visible holds step 4, a transaction opened in + `recording` that outlives one run's `--wait` holds the rerun's step 4 after a refusal, an + autovacuum worker on a loom table does not hold step 4 and a background worker with a + transaction open since `recording` does, a prepared transaction (after a restart with + `max_prepared_transactions` raised during the wait) makes step 4 refuse at once, naming + it, with the mode still `enforced` and a rerun refused until it is resolved and the + setting is back to 0, a write transaction open in a second database on the same server + holds step 4 and is named, a long read on the synchronous standby, or on a second standby, + with `hot_standby_feedback` on does not hold it, nor does a base backup running across it, + a write held open on a `replication=database` connection holds step 4, and so does a + `REPEATABLE READ` transaction on one, opened in `recording` and writing nothing before + `enforced`, a long logical dump (a `REPEATABLE READ` read-only transaction) on the cell's + database past `--wait` makes step 4 refuse, naming it, with the cell left `enforced`, and + the first cell's attach without `pg_read_all_stats` or with `max_prepared_transactions` + above 0 refuses before step 1 with the mode still `recording`, while a later cell's attach + does not require either; a missing or disabled org trigger is refused by attach and named + by `loom doctor`, a write for an unheld org on an `enforced` cell whose mode row was + deleted is refused and the state named by `loom doctor`; on that cell a pod with + `LOOM_CELL` unset is unready and runs no worker, and a CLI command declaring an org with + it unset is refused, as a pod is; updating, deleting or truncating the `first` row on an + `enforced` cell is refused, also under `session_replication_role = replica`, and a missing + or disabled `first`-row trigger is refused by attach and named by `loom doctor`; a running process with `LOOM_CELL` set on a cell whose mode row is removed answers 503 to a bearer request and runs no worker pass, and serves again after a rerun of `loom cell prepare` writes the row back while it runs; a cell the directory registers with no state, tenancy @@ -1548,12 +1550,12 @@ cell` store command refuses; `loom cell limit --closed` succeeds with the cell's rerun; a cell with both rows absent beside a tenancy row, and one beside only its own row, has the function refuse an unheld org (a boot creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the rows back; a - Postgres-only restore of an attached first cell to before its prepare is named by `loom -doctor` and closed by `loom cell restore`; a prepare rerun during a pending `loom cell -update` on an attached cell, with processes on both values and `loom.cell_self` lost, - writes both values back and keeps both sets of processes serving, and one before attach - refuses the two configurations; a prepare rerun on an attached cell whose `current` is - missing and whose pods' configuration differs from the directory row refuses, and one + Postgres-only restore of an attached first cell to before its prepare is named by + `loom doctor` and closed by `loom cell restore`; a prepare rerun during a pending + `loom cell update` on an attached cell, with processes on both values and `loom.cell_self` + lost, writes both values back and keeps both sets of processes serving, and one before + attach refuses the two configurations; a prepare rerun on an attached cell whose `current` + is missing and whose pods' configuration differs from the directory row refuses, and one before attach whose heartbeats report another bucket prefix refuses; a prepare rerun given a credential hash that is neither the row's `current` nor its `next` is refused; a prepare rerun on a `serving` later cell whose prefix holds data writes a removed mode row back; a @@ -1561,33 +1563,34 @@ update` on an attached cell, with processes on both values and `loom.cell_self` creates no trigger there), every process gated and `loom doctor` naming it, and a prepare rerun writes the row back; a later cell whose mode row is set to `recording` records nothing, its trigger refuses an unheld org, a process with `LOOM_CELL` set there serves as - in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, `loom -cell prepare` names the row, a CLI command declaring an org with `LOOM_CELL_ROUTER_TOKEN` - set refuses, naming it, and the router refuses its state read; a build N-1 process's - `memory.set` and delivery pass on a never-prepared deployment after N's migration succeed - and write no tenancy row, and on the first cell in `recording` a boot creates no org - trigger, and one created by hand reads the state and looks nothing up, a write under - `session_replication_role = replica` for an unheld org is skipped (origin triggers), an - `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is refused and one - that hits an existing row of it is not, a marked session's every write passes, and the - triggers on an `enforced` cell cover every table erasure classifies with an `org` column - but the three exclusions; and an org-keyed write while attach holds the mode table's lock - returns at once, with new-org requests answering within their deadline and mode-cache - reads returning at once throughout; a refusal at step 3's lock leaves the cell `recording` - with its directory row `enforced`, and the rerun runs step 2 again; a - `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by `loom -doctor`; `loom cell credential new` prints a credential of that form and its SHA-256, with - the directory and the stores unreachable; prepare and `--next` refuse an argument that is - not 64 lowercase hex characters, the credential and its bare 128-character secret - included, storing nothing; orgs a first cell records in `recording` with no data are - listed by `loom doctor`; a placement flood with no quota set is held to the rate limit at - each router replica, `loom doctor` names the cell as one with no quota, and a call refused - past it at one replica answers 503 with a `Retry-After` of at most 60 s, after which its - retry at that replica, with no other call of the cell admitted between, is admitted; a - cell's first prepare, with no directory row yet, passes the deployment-id check; a pod or - CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is refused by the - authority check; the first cell's existing tokens, refresh tokens and OAuth clients keep - working through the cutover, a logout through the shared host revokes its refresh token; + in `enforced`, a pod with `LOOM_CELL` unset there is unready and runs no worker, + `loom cell prepare` names the row, a CLI command declaring an org with + `LOOM_CELL_ROUTER_TOKEN` set refuses, naming it, and the router refuses its state read; a + build N-1 process's `memory.set` and delivery pass on a never-prepared deployment after + N's migration succeed and write no tenancy row, and on the first cell in `recording` a + boot creates no org trigger, and one created by hand reads the state and looks nothing up, + a write under `session_replication_role = replica` for an unheld org is skipped (origin + triggers), an `INSERT ... ON CONFLICT DO UPDATE` that inserts a row for an unheld org is + refused and one that hits an existing row of it is not, a marked session's every write + passes, and the triggers on an `enforced` cell cover every table erasure classifies with + an `org` column but the three exclusions; and an org-keyed write while attach holds the + mode table's lock returns at once, with new-org requests answering within their deadline + and mode-cache reads returning at once throughout; a refusal at step 3's lock leaves the + cell `recording` with its directory row `enforced`, and the rerun runs step 2 again; a + `LOOM_CELL_ROUTER_TOKEN` not of the `lcr_` form is refused at boot and named by + `loom doctor`; `loom cell credential new` prints a credential of that form and its + SHA-256, with the directory and the stores unreachable; prepare and `--next` refuse an + argument that is not 64 lowercase hex characters, the credential and its bare + 128-character secret included, storing nothing; orgs a first cell records in `recording` + with no data are listed by `loom doctor`; a placement flood with no quota set is held to + the rate limit at each router replica, `loom doctor` names the cell as one with no quota, + and a call refused past it at one replica answers 503 with a `Retry-After` of at most 60 + s, after which its retry at that replica, with no other call of the cell admitted between, + is admitted; a cell's first prepare, with no directory row yet, passes the deployment-id + check; a pod or CLI that lost `LOOM_PG_DSN` or `LOOM_STORAGE_AUTHORITY` on a cell is + refused by the authority check; the first cell's existing tokens, refresh tokens and OAuth + clients keep working through the cutover, a logout through the shared host revokes its + refresh token; - attach refuses a shared issuer, root secret, Keeper path, nested bucket prefix or configured org, a second cell that holds data, and a second attach while one runs; - an OIDC sign-in, an MCP client's authorization and a device flow for an org in the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 481b9a95ef..b713aefe0d 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -59,14 +59,14 @@ Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` o `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role -of `loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the -writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, -`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT -QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and `READ ON REMOTE`), no settings -profile or quota capping the rows or bytes it reads, and the insert into -`loom._migration_lock` that writes the probe nonce. The CLI needs write access to the -bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. +on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of +`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, +`SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, +`SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and +`READ ON REMOTE`), no settings profile or quota capping the rows or bytes it reads, and the +insert into `loom._migration_lock` that writes the probe nonce. The CLI needs write access +to the bucket's probe object and `LOOM_ROUTER_OPERATOR_TOKEN`. ## Run it @@ -80,13 +80,14 @@ loom cell attach --cell \ `--dry-run` runs the refusals and step 2's scan, prints the count and a sample of the orgs per source table, the ones it would skip, and whether the scan fit in half of `--scan-budget`, and writes nothing; the router's own checks at `attach/begin` (the probe, -the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object -and replica that may write the configured databases with `--ch-allow-user`, +the hosts, the upstream) do not run. Name every ClickHouse user, user directory, object and +replica that may write the configured databases with `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object` and `--ch-allow-replica`, and every Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each -list is audited by the router. `--wait` (default 600 s) bounds the waits together; -`--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of -it, since step 4 reads the same tables again. +list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; +every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs +one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, +and step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it From 78c2808934b8cde06ec59202673b3fbe88f3f485 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:05:39 +0000 Subject: [PATCH 029/181] Bound cell attach's mode commit and name booting processes Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +- docs/operations/cell-attach.md | 6 +- server/cli/cell_attach.py | 4 +- server/cli/cell_attach_pg.py | 34 +++++--- tests/integration/test_cell_attach.py | 4 +- tests/unit/test_cell_attach.py | 111 +++++++++++++++++++++++--- 6 files changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 53495c7a1d..f32815b635 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1065,8 +1065,10 @@ Then, in order (the step numbers are the ones the rest of this page uses): backed out. - **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the - cheap heartbeat refusals run before each try (a booting process skips the try). After a - failed try, the mode transaction an earlier run left in flight is found by its lock (only + cheap heartbeat refusals run before each try (a booting process skips the try). Each try, + including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and + a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell `enforced` and `attaching`, and a rerun resumes from the durable read. diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index b713aefe0d..ec6b6354ec 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -86,8 +86,10 @@ replica that may write the configured databases with `--ch-allow-user`, Postgres role besides attach's own that may make an org held with `--pg-allow-role`; each list is audited by the router. `--wait` (default 600 s) is the wall time the waits share; every lock try runs at least once, with a lock wait of at least 100 ms, so `--wait 0` runs -one try of each wait with no retry; `--scan-budget` (default 7200 s) bounds each full scan, -and step 2 must take at most half of it, since step 4 reads the same tables again. +one try of each wait with no retry. The final heartbeat checks get at least two heartbeats +plus the router timeout, with one extension for a process first seen booting; this floor +is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and +step 2 must take at most half of it, since step 4 reads the same tables again. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8ed70e2d37..6ef819796a 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -442,13 +442,13 @@ async def triggers(self) -> None: # -- step 3 ------------------------------------------------------------------------ - async def ready_for_commit(self) -> bool: + async def ready_for_commit(self) -> list[str]: """The cheap heartbeat refusals right before a mode-commit try: a refusal raises, a booting process skips the try.""" verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) if verdict.refusals: raise Refused(verdict.refusals) - return not verdict.booting + return verdict.booting async def commit_mode(self) -> None: from server.core import pg diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 0ca9fefcc1..60196363af 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -29,9 +29,13 @@ # The migration lock's wait a trigger try (a boot holds it for its migrations), in ms; a # try never waits longer than --wait has left. _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound +# A trigger try yields quickly to active writers. +_TRIGGER_LOCK_MS = 100 # not a setting: one try's bound # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait +# COMMIT's standby wait is outside Postgres's statement timeout. +_MODE_STANDBY_S = 5.0 # not a setting: one try's standby allowance # The server ends the lock session soon after its client is gone, releasing the lock. _KEEPALIVE = {"tcp_keepalives_idle": "30", "tcp_keepalives_interval": "10"} @@ -465,7 +469,7 @@ async def _ensure_table( yielded = False said = "" while True: - most_ms = 100 + most_ms = _TRIGGER_LOCK_MS holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True @@ -572,7 +576,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( conn: Any, confirm_lock: Callable[[], Any], - ready: Callable[[], Any], + ready: Callable[[], Awaitable[list[str]]], wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], @@ -581,17 +585,17 @@ async def commit_enforced( session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less when `wait` has less left), within `wait`; `confirm_lock` before each try and each termination (it refuses when another run holds the attach lock), and `ready` before - each try (False skips the try while a process boots). After a failed try the in-flight + each try (the booting process ids skip the try). After a failed try the in-flight mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` runs out; whether the commit landed is the durable read's to say.""" with wait.counting(): while True: await confirm_lock() - if await ready(): - mark("unknown") + booting = await ready() + if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS)) + done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -601,6 +605,13 @@ async def commit_enforced( log("the mode's lock is busy; retrying") await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): + if booting: + raise Refused( + [ + Refusal(f"process {p}", "still booting at the end of --wait") + for p in booting + ] + ) raise Refused( [ Refusal( @@ -612,15 +623,18 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int) -> bool: - """One try of the mode commit, its lock wait `lock_ms`; False when it timed out.""" +async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + """One try including COMMIT, bounded by its lock and statement waits plus the standby + allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with conn.transaction(): + async with asyncio.timeout(bound_s), conn.transaction(): await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" ) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 65064ab365..f6535cdb40 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -469,8 +469,8 @@ async def test_the_mode_commit_waits_out_a_for_share_holder(cell_stores) -> None await _prepare(cell_stores, _Network()) logged: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async with pg.dedicated() as holder, pg.dedicated() as conn: tx = holder.transaction() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 429c2fcfb7..95fc006333 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json from collections.abc import Callable from dataclasses import dataclass, field @@ -244,8 +245,8 @@ def lock(sql: str, *_a: Any) -> str: async def nothing() -> None: pass - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait @@ -652,8 +653,8 @@ def lock(*_a: Any) -> str: async def confirm() -> None: confirmed.append(True) - async def ready() -> bool: - return True + async def ready() -> list[str]: + return [] logged: list[str] = [] await apg.commit_enforced( @@ -684,24 +685,108 @@ async def confirm() -> None: @pytest.mark.asyncio +@pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( clock: _Clock, + monkeypatch: pytest.MonkeyPatch, + wait_s: int, ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.own_fp = (CURRENT, NEXT) + + async def read(_ch: Any) -> list[beats.Beat]: + return [_beat("a", cell_name=None), _beat("b", cell_name=None)] + + monkeypatch.setattr(beats, "read_beats", read) async def confirm() -> None: pass - async def booting() -> bool: - return False - - with pytest.raises(Refused, match="not taken within --wait"): + with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, confirm, booting, apg.Wait(2, clock), lambda _t: None, lambda _m: None + conn, + confirm, + run.ready_for_commit, + apg.Wait(wait_s, clock), + lambda _t: None, + lambda _m: None, ) + assert [str(r) for r in error.value.refusals] == [ + "process a: still booting at the end of --wait", + "process b: still booting at the end of --wait", + ] assert not any("LOCK TABLE" in s for s in conn.executed) +@pytest.mark.asyncio +@pytest.mark.parametrize("wait_s,bound_s", [(0, 8.1), (10, 10.0)]) +async def test_the_mode_try_bounds_a_commit_that_never_answers( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float +) -> None: + class _StalledCommit(FakeConn): + def transaction(self) -> Any: + conn = self + + class _Tx: + async def __aenter__(self) -> None: + conn.executed.append("BEGIN") + + async def __aexit__(self, *exc: Any) -> None: + assert not exc[0] + conn.executed.append("COMMIT") + await asyncio.Future() + + return _Tx() + + conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) + bounds: list[float] = [] + timeout = asyncio.timeout + + def bounded(seconds: float) -> Any: + bounds.append(seconds) + return timeout(0) + + monkeypatch.setattr(apg.asyncio, "timeout", bounded) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + apg.commit_enforced( + conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + ), + timeout=0.2, + ) + assert conn.executed[-1] == "COMMIT" + assert bounds == [bound_s] + assert marks == ["unknown"] + + +@pytest.mark.asyncio +async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: + conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) + marks: list[str] = [] + + async def confirm() -> None: + pass + + async def ready() -> list[str]: + return [] + + with pytest.raises(KeyboardInterrupt): + await apg.commit_enforced( + conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + ) + assert conn.executed[-1] == "ROLLBACK" + assert marks == [] + + @pytest.mark.asyncio async def test_the_durable_read_must_see_enforced_and_x_above_its_xid( monkeypatch: pytest.MonkeyPatch, @@ -921,8 +1006,8 @@ async def test_wait_0_runs_one_try_of_each_wait( ) marks: list[str] = [] - async def ok() -> bool: - return True + async def ok() -> list[str]: + return [] async def nothing() -> None: pass @@ -975,10 +1060,10 @@ def lock(*_a: Any) -> str: async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} - async def ready(_self: Any) -> bool: + async def ready(_self: Any) -> list[str]: if refused_before_any_try: raise Refused([Refusal("process p1", "runs with LOOM_CELL unset")]) - return True + return [] async def confirm(_self: Any) -> None: pass From 62c6b4783ac06200c31d32f735def75e48da017a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:18:57 +0000 Subject: [PATCH 030/181] Require every cell probe grant before attach Check each probe privilege separately because a comma-separated privilege list accepts any one of them. Run the attach integration tests in the Postgres matrix and limit their heartbeat reads to the test's own process IDs while retaining the real queries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/cli/cell_attach_pg.py | 5 ++- tests/integration/test_cell_attach.py | 50 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 564393a0f7..22342c2cab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -898,6 +898,7 @@ jobs: tests/integration/test_router_directory_pg.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py + tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 60196363af..583d99775d 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -397,7 +397,10 @@ async def session_passes() -> bool: ('SELECT on loom.authority', has_table_privilege('loom.authority', 'SELECT')), ('INSERT on loom.tenancy', has_table_privilege('loom.tenancy', 'INSERT')), ('SELECT, INSERT, UPDATE and DELETE on loom.cell_probe', - has_table_privilege('loom.cell_probe', 'SELECT, INSERT, UPDATE, DELETE')), + has_table_privilege('loom.cell_probe', 'SELECT') + AND has_table_privilege('loom.cell_probe', 'INSERT') + AND has_table_privilege('loom.cell_probe', 'UPDATE') + AND has_table_privilege('loom.cell_probe', 'DELETE')), ('TEMPORARY on the database', has_database_privilege(current_database(), 'TEMPORARY')) ) v(need, held) WHERE NOT held diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index f6535cdb40..aaa48b8f0e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -102,6 +102,21 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str ch = get_client() await ch.connect() ids: list[str] = [] + real_live, real_recent = instances.read_live, instances.read_recent + + async def read_live(ch: Any, *, all_deployments: bool = False) -> list[dict[str, Any]]: + rows = await real_live(ch, all_deployments=all_deployments) + return [r for r in rows if r["instance_id"] in ids] + + async def read_recent( + ch: Any, *, window_s: float, since: str | None = None + ) -> list[dict[str, Any]]: + rows = await real_recent(ch, window_s=window_s, since=since) + return [r for r in rows if r["instance_id"] in ids] + + # The integration server shares ClickHouse and reports its own process heartbeats. + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: await pg_migrate.migrate() await pg.execute( @@ -669,6 +684,41 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_attach_requires_every_probe_privilege(cell_stores) -> None: + role = f"it_probe_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + privileges = ("SELECT", "INSERT", "UPDATE", "DELETE") + grants = [ + ("SELECT",), + *(tuple(p for p in privileges if p != missing) for missing in privileges), + privileges, + ] + observed: dict[tuple[str, ...], list[str]] = {} + try: + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + for held in grants: + await conn.execute(f"REVOKE ALL ON loom.cell_probe FROM {role}") + await conn.execute(f"GRANT {', '.join(held)} ON loom.cell_probe TO {role}") + await conn.execute(f"SET ROLE {role}") + refusals = await cell_attach_pg.own_role_refusals(conn, []) + await conn.execute("RESET ROLE") + observed[held] = [r.reason for r in refusals if "loom.cell_probe" in r.reason] + assert observed == { + held: ( + [] + if held == privileges + else ["attach's role lacks SELECT, INSERT, UPDATE and DELETE on loom.cell_probe"] + ) + for held in grants + } + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_any_write( cell_stores, From 95bb74c0fe04dd0981970e9305c4ff447cb5d703 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:29:02 +0000 Subject: [PATCH 031/181] Keep cell attach deadlines from waiting on rollback Manage mode and direct-session transactions explicitly so a canceled statement cannot start an unbounded rollback. Roll back ordinary mode statement errors within the try's deadline; terminate synchronously when a try ends with an error or cancellation. Signed-off-by: zlareb1 --- server/cli/cell_attach_pg.py | 64 +++++++++++------- tests/unit/test_cell_attach.py | 116 ++++++++++++++++++++++++++++----- 2 files changed, 139 insertions(+), 41 deletions(-) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 583d99775d..d58b553d7a 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -122,10 +122,18 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: startup parameters drops it). Behind a transaction pooler a session-level lock would outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" - async with conn.transaction(): - first = await conn.fetchrow(_IDENTITY) - async with conn.transaction(): - second = await conn.fetchrow(_IDENTITY) + identities = [] + try: + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() + except BaseException: + # A cancelled read must not wait for a transaction context's rollback. + conn.terminate() + raise + first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): out.append( @@ -627,31 +635,39 @@ async def commit_enforced( async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: - """One try including COMMIT, bounded by its lock and statement waits plus the standby - allowance; False on a statement timeout, TimeoutError if COMMIT never answers.""" + """One try including rollback or COMMIT, bounded by its lock and statement waits plus + the standby allowance; False after a statement timeout rolls back, TimeoutError at + the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S try: - async with asyncio.timeout(bound_s), conn.transaction(): - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + async with asyncio.timeout(bound_s): + tx = conn.transaction() + await tx.start() + try: + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute( + "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" + ) + if status != "UPDATE 1": + raise Refused( + [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] + ) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False + raise + await tx.commit() return True - except Refused: + except BaseException: + # Termination is synchronous: cancellation never starts an unbounded rollback. + conn.terminate() raise - except Exception as e: - if getattr(e, "sqlstate", None) not in _TIMED_OUT: - raise - return False async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 95fc006333..065f3a455d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -473,14 +473,29 @@ def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await conn.execute("BEGIN") + + async def commit(self) -> None: + await conn.execute("COMMIT") + + async def rollback(self) -> None: + await conn.execute("ROLLBACK") + async def __aenter__(self) -> None: - conn.executed.append("BEGIN") + await self.start() async def __aexit__(self, *exc: Any) -> None: - conn.executed.append("ROLLBACK" if exc[0] else "COMMIT") + if exc[0]: + await self.rollback() + else: + await self.commit() return _Tx() + def terminate(self) -> None: + self.executed.append("TERMINATE") + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -663,6 +678,8 @@ async def ready() -> list[str]: assert logged == ["the mode's lock is busy; retrying"] # Before each try and the one termination. assert len(confirmed) == 3 + assert conn.executed.count("ROLLBACK") == 1 + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio @@ -725,19 +742,11 @@ async def test_the_mode_try_bounds_a_commit_that_never_answers( monkeypatch: pytest.MonkeyPatch, clock: _Clock, wait_s: int, bound_s: float ) -> None: class _StalledCommit(FakeConn): - def transaction(self) -> Any: - conn = self - - class _Tx: - async def __aenter__(self) -> None: - conn.executed.append("BEGIN") - - async def __aexit__(self, *exc: Any) -> None: - assert not exc[0] - conn.executed.append("COMMIT") - await asyncio.Future() - - return _Tx() + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "COMMIT": + await asyncio.Future() + return result conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] @@ -763,11 +772,84 @@ async def ready() -> list[str]: ), timeout=0.2, ) - assert conn.executed[-1] == "COMMIT" + assert "COMMIT" in conn.executed + assert conn.executed[-1] == "TERMINATE" assert bounds == [bound_s] assert marks == ["unknown"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["LOCK TABLE", "UPDATE", "ROLLBACK"]) +async def test_a_mode_deadline_never_waits_for_rollback( + monkeypatch: pytest.MonkeyPatch, stall: str +) -> None: + class _StalledRollback(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "ROLLBACK" and sql.startswith("LOCK TABLE"): + raise _Timeout() + if sql.startswith(stall) or sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + marks: list[str] = [] + task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "mode deadline is stuck awaiting rollback" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + assert marks == (["unknown"] if stall == "UPDATE" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_read", [1, 2]) +@pytest.mark.parametrize("failure", ["cancel", "error"]) +async def test_a_failed_direct_session_read_never_waits_for_rollback( + failed_read: int, failure: str +) -> None: + class _StalledRollback(FakeConn): + reads = 0 + + async def fetchrow(self, sql: str, *args: Any) -> Any: + self.reads += 1 + if self.reads == failed_read: + if failure == "error": + raise ValueError("identity read failed") + await asyncio.Future() + return (1, "t0") + + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if sql == "ROLLBACK": + await asyncio.Future() + return result + + conn = _StalledRollback() + task = asyncio.create_task(apg.direct_refusals(conn, "attach")) + await asyncio.sleep(0) + if failure == "cancel": + task.cancel() + done, _ = await asyncio.wait([task], timeout=0.2) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert task in done, "direct-session failure is stuck awaiting rollback" + if failure == "cancel": + assert task.cancelled() + else: + assert isinstance(task.exception(), ValueError) + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + @pytest.mark.asyncio async def test_an_interrupt_waiting_for_the_mode_lock_leaves_recording(clock: _Clock) -> None: conn = FakeConn([("LOCK TABLE", KeyboardInterrupt())]) @@ -783,7 +865,7 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "ROLLBACK" + assert conn.executed[-1] == "TERMINATE" assert marks == [] From d5ec3b9b118b5943c500f0169323d8569ceb4f4d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:44:09 +0000 Subject: [PATCH 032/181] Bound attach trigger attempts and scope gate cleanup to its run Give each trigger attempt a client deadline covering migration lock acquisition, DDL, commit, rollback and unlock. Use explicit transactions so cancellation terminates the session without waiting for rollback or masking the original error with lock cleanup. Give gate batch scopes a unique prefix per run so cleanup preserves existing rows. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- server/cli/cell_attach_pg.py | 47 ++++++++------ server/core/cell_triggers.py | 21 +++++- tests/unit/test_bench_cell_trigger_gate.py | 53 +++++++++++++++ tests/unit/test_cell_attach.py | 75 ++++++++++++++++++++++ tests/unit/test_cell_org_triggers.py | 14 ++-- 6 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_bench_cell_trigger_gate.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 4363a810d6..8797b4b8e7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -80,8 +80,9 @@ async def _batch(args: argparse.Namespace, marked: bool) -> float: t = time.perf_counter() await conn.execute( "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, 'gate-' || g || '-' || $2 FROM generate_series(1, 1000) g", + "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", args.org, + args.scope_prefix, uuid.uuid4().hex[:8], ) times.append((time.perf_counter() - t) * 1000) @@ -98,6 +99,7 @@ async def main() -> None: p.add_argument("--org", required=True) p.add_argument("--namespace", required=True) args = p.parse_args() + args.scope_prefix = f"gate-{uuid.uuid4().hex}-" try: for _ in range(ROUNDS): for on in (True, False): @@ -120,7 +122,9 @@ async def _clean(args: argparse.Namespace) -> None: conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) try: await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND scope LIKE 'gate-%'", args.org + "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", + args.org, + args.scope_prefix, ) finally: await conn.close() diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d58b553d7a..2af085f86f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -31,6 +31,7 @@ _MIGRATION_LOCK_MS = 10_000 # not a setting: one try's bound # A trigger try yields quickly to active writers. _TRIGGER_LOCK_MS = 100 # not a setting: one try's bound +_TRIGGER_TRY_MARGIN_S = 8.0 # not a setting: catalog, DDL and standby allowance # The mode commit's lock wait a try, in ms, and its statement bound's margin above it. _MODE_LOCK_MS = 2_000 # not a setting: one try's bound _MODE_STATEMENT_MARGIN_MS = 3_000 # not a setting: a margin over the lock wait @@ -524,29 +525,37 @@ async def _migration_holder(conn: Any, key: int) -> str: async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the - try floor. A busy migration lock answers `busy` naming that lock's holder.""" + try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or + rollback. A busy migration lock answers `busy` naming that lock's holder.""" from server.core.schema.pg_migrate import _LOCK_KEY - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + bound_s = ( + wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) + ) / 1000 + _TRIGGER_TRY_MARGIN_S try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) == "55P03": - # A boot migrating: the next try. - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + async with asyncio.timeout(bound_s): + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") + try: + await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) + return outcome + except BaseException: + # Closing the session releases its locks without asynchronous error cleanup. + conn.terminate() raise - finally: - await conn.execute("RESET lock_timeout") - try: - return await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - finally: - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 7a1ee5e5b7..1899a46bab 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -328,18 +328,32 @@ async def _run_ddl( import asyncpg for attempt in range(attempts): + rolled_back = False try: - async with conn.transaction(): + tx = conn.transaction() + await tx.start() + try: await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) + except Exception: + await tx.rollback() + rolled_back = True + raise + await tx.commit() return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: + if not rolled_back: + conn.terminate() + raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e ) return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: + if not rolled_back: + conn.terminate() + raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: await asyncio.sleep(LOCK_RETRY_S) @@ -354,6 +368,11 @@ async def _run_ddl( held, ) return Outcome("busy", held) + except BaseException: + if not rolled_back: + # Cancellation must not wait for another statement on this session. + conn.terminate() + raise return Outcome("busy") diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py new file mode 100644 index 0000000000..89f32cddde --- /dev/null +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -0,0 +1,53 @@ +"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" + +from __future__ import annotations + +import argparse +import sqlite3 +from typing import Any + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.asyncio +async def test_batch_cleanup_preserves_preexisting_gate_scopes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = sqlite3.connect(":memory:") + db.execute("ATTACH DATABASE ':memory:' AS loom") + db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") + db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) + prefix = "gate-own-run-" + sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] + db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) + + class _Conn: + async def execute(self, sql: str, *args: Any) -> None: + sql = sql.replace( + "FROM generate_series(1, 1000) g", + "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " + "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", + ) + db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) + + async def close(self) -> None: + pass + + async def connect(*_a: Any, **_kw: Any) -> Any: + return _Conn() + + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate, "INSERTS", 1) + args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) + try: + await gate._batch(args, True) + assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 + await gate._clean(args) + assert ( + db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() + == sentinel + ) + finally: + db.close() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 065f3a455d..6d57cbb740 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -1038,6 +1038,81 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: assert asked == [f"{apg.TRY_FLOOR_MS}ms"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["migration", "reset", "ddl", "commit", "rollback", "unlock"]) +async def test_a_trigger_try_deadline_covers_transaction_and_lock_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, stall: str +) -> None: + import asyncpg + + from server.core import cell_triggers + + class _Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + result = await super().execute(sql, *args) + if stall == "rollback" and sql == "CREATE TRIGGER stub": + raise asyncpg.LockNotAvailableError("busy") + needles = { + "migration": "SELECT pg_advisory_lock", + "reset": "RESET lock_timeout", + "ddl": "CREATE TRIGGER stub", + "commit": "COMMIT", + "rollback": "ROLLBACK", + "unlock": "SELECT pg_advisory_unlock", + } + if sql.startswith(needles[stall]) or (stall == "ddl" and sql == "ROLLBACK"): + await asyncio.Future() + return result + + async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: + return await cell_triggers._run_ddl( + conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False + ) + + timeout = asyncio.timeout + monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = _Stalled() + task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + done, _ = await asyncio.wait([task], timeout=0.2) + try: + assert task in done, f"trigger deadline is stuck at {stall}" + assert isinstance(task.exception(), TimeoutError) + assert conn.executed[-1] == "TERMINATE" + if stall in {"migration", "ddl", "commit", "rollback"}: + assert apg._MIGRATION_UNLOCK not in conn.executed + finally: + for _ in range(3): + if task.done(): + break + task.cancel() + await asyncio.wait([task], timeout=0.01) + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["migration", "ddl"]) +async def test_a_failed_trigger_try_does_not_mask_its_error_with_cleanup( + monkeypatch: pytest.MonkeyPatch, clock: _Clock, failed: str +) -> None: + from server.core import cell_triggers + + async def ensure_table(*_a: Any, **_kw: Any) -> Any: + raise ValueError("DDL failed") + + monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) + conn = FakeConn( + [ + ("pg_advisory_lock", ValueError("migration failed") if failed == "migration" else "OK"), + ("RESET", RuntimeError("reset failed") if failed == "migration" else "OK"), + ("pg_advisory_unlock", RuntimeError("unlock failed")), + ] + ) + with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): + await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + assert conn.executed[-1] == "TERMINATE" + + @pytest.mark.asyncio async def test_a_busy_migration_lock_is_named_as_such( monkeypatch: pytest.MonkeyPatch, clock: _Clock diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index decb365521..769cfe1bb3 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -617,11 +617,17 @@ def __init__(self, error: Exception) -> None: self.error = error def transaction(self) -> Any: - @contextlib.asynccontextmanager - async def _tx() -> AsyncIterator[None]: - yield + class _Tx: + async def start(self) -> None: + pass - return _tx() + async def commit(self) -> None: + pass + + async def rollback(self) -> None: + pass + + return _Tx() async def fetchrow(self, _sql: str, *_a: Any) -> dict[str, Any]: return { From 20681aa183677097ec2c1e6a31299a14d4230ed4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 20:57:56 +0000 Subject: [PATCH 033/181] Check member-role privileges and repeat attach checks after placement Count privileges a login can reach through SET ROLE or an ADMIN regrant, including noninherited group grants. Keep the owner exemptions and explicit role allow list. Repeat the writer, trigger, Postgres catalog and standby checks after the rescan and placement before marking the cell serving. Retain the heartbeat watch and its final wait floor. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 3 +- server/cli/cell_attach.py | 11 ++-- server/cli/cell_attach_pg.py | 8 ++- tests/integration/test_cell_attach.py | 87 +++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index f32815b635..dd1189b696 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1093,7 +1093,8 @@ Then, in order (the step numbers are the ones the rest of this page uses): source in full, Postgres before ClickHouse (the outbox and delivery tables move rows to ClickHouse), and the queue and drain tables and the tables a view reads before the others (`cell_scan.MOVE_SOURCES`); the tenancy rows; the placements, in `attach/place` batches; - the writer check again; and the final checks, under a floor of their own (the larger of + the writer, trigger, Postgres and standby checks again; and the final checks, under a + floor of their own (the larger of what `--wait` left and `instances.boot_allowance_s()`): the heartbeats, read through the rescan every half of the time the leader keeps a stopped row (`instances.PRUNE_AFTER_S`), refusing any process that ran with `LOOM_CELL` unset during it, `loom.cell_self` equal to diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 6ef819796a..bf4e81521e 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -516,7 +516,7 @@ async def durable(self) -> int: # -- step 4 ------------------------------------------------------------------------ - async def cheap_checks(self, stopped_unset: str) -> None: + async def nonheartbeat_checks(self) -> None: from server.core import pg_profile out, _, _ = await self.writer_refusals() @@ -527,10 +527,13 @@ async def cheap_checks(self, stopped_unset: str) -> None: out += await self.catalog_refusals() if out: raise Refused(out) - await self.check_beats(stopped_unset) if (now := await pg_profile.standby_now()) != "synchronous": raise _refused("postgres standbys", f"commits are acknowledged as {now}") + async def cheap_checks(self, stopped_unset: str) -> None: + await self.nonheartbeat_checks() + await self.check_beats(stopped_unset) + async def final_checks(self, watch: beats.Watch) -> None: from server.core import instances @@ -575,9 +578,7 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: ) placed += int(answer.get("placed") or 0) self.report.update(orgs_found=len(orgs), placed_now=placed, skipped=self._named(found)) - writers, _, _ = await self.writer_refusals() - if writers: - raise Refused(writers) + await self.nonheartbeat_checks() finally: task.cancel() with contextlib.suppress(asyncio.CancelledError): diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 2af085f86f..56601e72e9 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -231,6 +231,7 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: # Every right a login role may hold that lets it make an org held, lower the mode, or run # its own SQL as the servers on their marked sessions: each row a role and why. +# Membership can supply a right through SET ROLE or an ADMIN regrant without INHERIT. _RIGHTS = f""" WITH owned AS ( SELECT c.relowner AS owner, 'owns loom.' || c.relname AS why @@ -248,7 +249,8 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: SELECT n.nspowner, 'owns schema ' || n.nspname FROM pg_namespace n WHERE {_SYSTEM_SCHEMA} ), login AS (SELECT oid, rolname FROM pg_roles WHERE rolcanlogin) -SELECT r.rolname AS role, w.why FROM login r, LATERAL ( +SELECT DISTINCT l.rolname AS role, w.why FROM login l +JOIN pg_roles r ON pg_has_role(l.oid, r.oid, 'MEMBER'), LATERAL ( SELECT 'may insert into loom.tenancy' WHERE has_any_column_privilege(r.oid, 'loom.tenancy', 'INSERT') UNION ALL SELECT 'may update loom.tenancy (org)' @@ -273,10 +275,10 @@ async def encoding_refusals(conn: Any) -> list[Refusal]: UNION ALL SELECT 'is a member of ' || g.rolname FROM pg_roles g WHERE g.rolname IN ('pg_write_server_files', 'pg_execute_server_program') AND pg_has_role(r.oid, g.oid, 'MEMBER') - UNION ALL SELECT o.why || CASE WHEN o.owner = r.oid THEN '' + UNION ALL SELECT o.why || CASE WHEN o.owner = l.oid THEN '' ELSE ' (as a member of ' || pg_get_userbyid(o.owner) || ')' END - FROM owned o WHERE pg_has_role(r.oid, o.owner, 'MEMBER') + FROM owned o WHERE pg_has_role(l.oid, o.owner, 'MEMBER') ) w(why) ORDER BY 1, 2 """ diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index aaa48b8f0e..168008dfa8 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -324,6 +324,41 @@ async def durable_then_unset(self: Any) -> int: assert (await _row()).state == "serving" +@pytest.mark.asyncio +@pytest.mark.parametrize("defect", ["trigger", "catalog"]) +async def test_a_defect_during_the_rescan_keeps_the_cell_attaching( + cell_stores, monkeypatch, capsys, defect: str +) -> None: + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_rescan_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_scan = cell_attach.Run.scan + + async def scan_then_defect(self: Any, *args: Any) -> Any: + found = await real_scan(self, *args) + if self.mode == "enforced": + if defect == "trigger": + await pg.execute( + f"ALTER TABLE loom.scope_generations DISABLE TRIGGER {cell_triggers.INSERT_TRIGGER}" + ) + else: + await pg.execute(f"GRANT INSERT (org) ON loom.tenancy TO {role}") + return found + + monkeypatch.setattr(cell_attach.Run, "scan", scan_then_defect) + try: + assert await _attach(stores, network) == 1 + refused = capsys.readouterr().err + assert ("postgres triggers" if defect == "trigger" else f"postgres role {role}") in refused + assert await _mode() == "enforced" + assert (await _row()).state == "attaching" + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_a_second_run_is_refused_at_the_lock(cell_stores, capsys) -> None: stores, network = cell_stores, _Network() @@ -505,6 +540,58 @@ async def release() -> None: assert logged == ["the mode's lock is busy; retrying"] +@pytest.mark.asyncio +@pytest.mark.parametrize("membership", ["set", "admin"]) +async def test_a_login_can_lower_mode_through_a_noninherited_group( + cell_stores, membership: str +) -> None: + await _prepare(cell_stores, _Network()) + group = f"it_group_{uuid.uuid4().hex[:6]}" + login = f"it_login_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {group} NOLOGIN") + try: + await _admin(f"CREATE ROLE {login} LOGIN") + async with pg.dedicated() as conn: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {group}") + await conn.execute(f"GRANT UPDATE (mode) ON loom.tenancy_mode TO {group}") + flags = ( + "ADMIN FALSE, INHERIT FALSE, SET TRUE" + if membership == "set" + else "ADMIN TRUE, INHERIT FALSE, SET FALSE" + ) + await conn.execute(f"GRANT {group} TO {login} WITH {flags}") + assert not await conn.fetchval( + "SELECT has_column_privilege($1, 'loom.tenancy_mode', 'mode', 'UPDATE')", login + ) + holders = await cell_attach_pg.privilege_holders(conn, exempt_self=True) + await conn.execute(f"SET SESSION AUTHORIZATION {login}") + if membership == "admin": + await conn.execute(f"GRANT {group} TO {login} WITH SET TRUE") + await conn.execute(f"SET ROLE {group}") + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced'") == "UPDATE 1" + ) + assert ( + await conn.execute("UPDATE loom.tenancy_mode SET mode = 'recording'") == "UPDATE 1" + ) + await conn.execute("RESET SESSION AUTHORIZATION") + assert "may update loom.tenancy_mode (mode)" in holders.others.get(login, []) + assert login not in holders.own + assert any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, []) + ) + assert not any( + r.subject == f"postgres role {login}" + for r in cell_attach_pg.privilege_refusals(holders, [login]) + ) + finally: + async with pg.dedicated() as conn: + await conn.execute(f"DROP OWNED BY {login}, {group}") + await _admin(f"DROP ROLE IF EXISTS {login}") + await _admin(f"DROP ROLE {group}") + + @pytest.mark.asyncio @pytest.mark.parametrize( "grant,why", From cb95fffcde72511d555dc055d5e3b0f1a3ccbb08 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:12:45 +0000 Subject: [PATCH 034/181] Require effective predefined role privileges before attach Check USAGE of pg_read_all_stats and pg_signal_backend because attach uses their privileges without SET ROLE. Keep MEMBER for the separate untrusted-writer reachability checks. Refuse noninherited capabilities before attach writes, and explain the inherited grant requirement in the refusal labels and runbook. Test both inherited neighbors and signaling a scratch backend. Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 3 +- server/cli/cell_attach_pg.py | 6 ++- tests/integration/test_cell_attach.py | 77 +++++++++++++++++++++++++++ tests/unit/test_cell_attach.py | 4 +- 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index ec6b6354ec..5cf6f74350 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -55,7 +55,8 @@ Every item below is a precondition attach checks, or one it cannot check and rel `max_prepared_transactions = 0`; the database encoding is UTF8. 13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. -Attach's Postgres role needs `pg_read_all_stats`, `pg_signal_backend`, `USAGE` on schema +Attach's Postgres role needs the privileges of `pg_read_all_stats` and `pg_signal_backend` +without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on schema `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 56601e72e9..b8cc9e6e44 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -398,8 +398,10 @@ async def session_passes() -> bool: _OWN_GRANTS = """ SELECT need, held FROM (VALUES - ('membership in pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER')), - ('membership in pg_signal_backend', pg_has_role(current_user, 'pg_signal_backend', 'MEMBER')), + ('effective privileges of pg_read_all_stats', + pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')), + ('effective privileges of pg_signal_backend', + pg_has_role(current_user, 'pg_signal_backend', 'USAGE')), ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 168008dfa8..894635a4d9 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -771,6 +771,83 @@ async def pg_orgs(table: str, budget: Any) -> list[str]: assert grade == "WARN" and detail.endswith(": loom.it_large") +@pytest.mark.asyncio +async def test_signaling_another_role_requires_an_effective_signal_grant(cell_stores) -> None: + import asyncpg + + caller = f"it_signal_{uuid.uuid4().hex[:6]}" + target = f"it_target_{uuid.uuid4().hex[:6]}" + password = uuid.uuid4().hex + await _admin(f"CREATE ROLE {caller} LOGIN") + try: + await _admin(f"CREATE ROLE {target} LOGIN PASSWORD '{password}'") + victim = await asyncpg.connect(settings.pg_dsn, user=target, password=password) + try: + pid = await victim.fetchval("SELECT pg_backend_pid()") + async with pg.dedicated() as admin, pg.dedicated() as signal: + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT FALSE") + await signal.execute(f"SET SESSION AUTHORIZATION {caller}") + with pytest.raises(asyncpg.InsufficientPrivilegeError): + await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + await admin.execute(f"GRANT pg_signal_backend TO {caller} WITH INHERIT TRUE") + assert await signal.fetchval("SELECT pg_terminate_backend($1)", pid) + finally: + await victim.close() + finally: + await _admin(f"DROP ROLE {caller}") + await _admin(f"DROP ROLE IF EXISTS {target}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capability", ["pg_read_all_stats", "pg_signal_backend"]) +async def test_attach_needs_its_predefined_capabilities_without_set_role( + cell_stores, monkeypatch, capsys, capability: str +) -> None: + from contextlib import asynccontextmanager + + stores, network = cell_stores, _Network() + await _prepare(stores, network) + role = f"it_caps_{uuid.uuid4().hex[:6]}" + await _admin(f"CREATE ROLE {role} LOGIN") + real_dedicated = pg.dedicated + + @asynccontextmanager + async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: + async with real_dedicated(**kwargs) as conn: + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + yield conn + + try: + async with real_dedicated() as conn: + owner = await conn.fetchval("SELECT current_user") + await conn.execute(f"GRANT {owner} TO {role} WITH INHERIT TRUE, SET FALSE") + await conn.execute(f"GRANT pg_read_all_stats, pg_signal_backend TO {role}") + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT FALSE") + assert await conn.fetchval("SELECT pg_has_role($1, $2, 'MEMBER')", role, capability) + assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) + before = await _row() + with monkeypatch.context() as own: + own.setattr(pg, "dedicated", as_attacher) + assert await _attach(stores, network) == 1 + assert ( + f"attach's role lacks effective privileges of {capability}" in capsys.readouterr().err + ) + after = await _row() + assert (after.state, after.mode, after.placed) == (before.state, before.mode, before.placed) + assert await _mode() == "recording" + assert await pg.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + async with real_dedicated() as conn: + await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT TRUE") + with monkeypatch.context() as own: + own.setattr(pg, "dedicated", as_attacher) + assert await _attach(stores, network) == 0, capsys.readouterr().err + assert (await _row()).state == "serving" + finally: + async with real_dedicated() as conn: + await conn.execute(f"DROP OWNED BY {role}") + await _admin(f"DROP ROLE {role}") + + @pytest.mark.asyncio async def test_attach_requires_every_probe_privilege(cell_stores) -> None: role = f"it_probe_{uuid.uuid4().hex[:6]}" diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 6d57cbb740..c0054d0c9e 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -572,7 +572,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: conn = FakeConn( [ ("SELECT current_user", "attacher"), - ("WHERE NOT held", [{"need": "membership in pg_read_all_stats"}]), + ("WHERE NOT held", [{"need": "effective privileges of pg_read_all_stats"}]), ("has_table_privilege($1, 'SELECT')", lambda t: t != "loom.memory_current"), ("max_prepared_transactions", 5), ] @@ -586,7 +586,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ] refusals = await apg.own_role_refusals(conn, tables) assert [str(r) for r in refusals] == [ - "postgres role attacher: attach's role lacks membership in pg_read_all_stats", + "postgres role attacher: attach's role lacks effective privileges of pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", "postgres role attacher: attach's role lacks membership in the owner role of " "loom.scope_generations", From fde3798cfd2e226d53b20393274b801c58b44ddf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:18:35 +0000 Subject: [PATCH 035/181] Describe the effective owner privileges attach requires Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 5 +++-- server/cli/cell_attach_pg.py | 9 +++++---- tests/integration/test_cell_attach.py | 5 ++++- tests/unit/test_cell_attach.py | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 5cf6f74350..e674da2997 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -60,8 +60,9 @@ without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on sch `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` -on the database, `EXECUTE` on `pg_logical_emit_message`, and membership in the owner role of -`loom.tenancy` and of every guarded table. Its ClickHouse user needs the grants the writer +on the database, `EXECUTE` on `pg_logical_emit_message`, and effective privileges of the owner +role of `loom.tenancy` and every guarded table without `SET ROLE` (grant their memberships +with `INHERIT TRUE`). Its ClickHouse user needs the grants the writer check names (`SELECT` on `system.*` and on the configured databases, `SHOW USERS`, `SHOW ROLES`, `SHOW ROW POLICIES`, `SHOW TABLES`, `SHOW COLUMNS`, `SYSTEM FLUSH ASYNC INSERT QUEUE`, and on a cluster `SYSTEM SYNC REPLICA`, `CLUSTER` and diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index b8cc9e6e44..12e1808f8b 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -422,8 +422,8 @@ async def session_passes() -> bool: async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: """What attach's own role lacks, `tables` being the rows `cell_triggers.ORG_TABLES` - read: its grants, `SELECT` on every org table, and membership in the owner role of every - table it gives triggers (or the trigger pass could not alter it); and a server that + read: its grants, `SELECT` on every org table, and effective owner-role privileges for + every table it gives triggers (or the trigger pass could not alter it); and a server that allows prepared transactions.""" role = await conn.fetchval("SELECT current_user") lacks = [str(r["need"]) for r in await conn.fetch(_OWN_GRANTS)] @@ -435,7 +435,7 @@ async def own_role_refusals(conn: Any, tables: list[Any]) -> list[Refusal]: planned = set(cell_triggers.planned_names(tables)) for r in tables: if r["relname"] in planned and not r["owned"]: - lacks.append(f"membership in the owner role of loom.{r['relname']}") + lacks.append(f"effective privileges of the owner role of loom.{r['relname']}") out = [Refusal(f"postgres role {role}", f"attach's role lacks {n}") for n in lacks] out += await prepared_refusals(conn) return out @@ -495,7 +495,8 @@ async def _ensure_table( if outcome.answer == "denied": return Refusal( f"postgres loom.{table}", - "attach's role cannot alter it: make the role a member of its owner role", + "attach's role cannot alter it: grant inherited privileges of its owner role " + "without SET ROLE", ) if not await wait.pause(): if outcome.holders.startswith(_MIGRATION_HELD): diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 894635a4d9..3450558fe4 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -902,7 +902,10 @@ async def test_a_role_with_the_grants_but_not_the_owner_role_is_refused_before_a rows = list(await conn.fetch(cell_triggers.ORG_TABLES)) refusals = await cell_attach_pg.own_role_refusals(conn, rows) lacking = [r.reason for r in refusals if "owner role" in r.reason] - assert "attach's role lacks membership in the owner role of loom.tenancy" in lacking + assert ( + "attach's role lacks effective privileges of the owner role of loom.tenancy" + in lacking + ) assert any("loom.scope_generations" in r for r in lacking) # The trigger pass tells the cause apart: denied, not a busy lock. outcome = await cell_triggers.ensure_table( diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index c0054d0c9e..0ecd3a96b3 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -579,7 +579,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: ) tables = [ {"relname": "memory_current", "relkind": "r", "owned": True}, - # Its grants are held, but not its owner role's membership: the trigger pass could + # Its grants are held, but not its effective owner-role privileges: the pass could # not alter it, so it refuses before any write. {"relname": "scope_generations", "relkind": "r", "owned": False}, {"relname": "tenancy", "relkind": "r", "owned": True}, @@ -588,7 +588,7 @@ async def test_own_role_lacks_and_prepared_transactions_refuse() -> None: assert [str(r) for r in refusals] == [ "postgres role attacher: attach's role lacks effective privileges of pg_read_all_stats", "postgres role attacher: attach's role lacks SELECT on loom.memory_current", - "postgres role attacher: attach's role lacks membership in the owner role of " + "postgres role attacher: attach's role lacks effective privileges of the owner role of " "loom.scope_generations", "postgres max_prepared_transactions: is 5: a prepared transaction outlives every " "session, so step 4 could not wait for it; set it to 0", From cad839bdf5051753a10c7f162e21e104f7b17ef4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:24:41 +0000 Subject: [PATCH 036/181] Build placement indexes outside migration transactions Concurrent index builds release the ordinary migration key so a shipped blocking migrator can finish without retaining a snapshot behind the build. Retry from the catalogs and checksum ledger, and acknowledge a WAL fence on recorded reruns. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ docs/development/migrations.md | 54 ++- .../004_placements_by_cell_org.sql | 3 + server/core/schema/pg_migrate.py | 154 ++++++-- server/core/schema/pg_online_index.py | 125 ++++++ .../fixtures/pg_migrate_n_minus_one.py | 151 +++++++ tests/integration/test_pg_online_index.py | 373 ++++++++++++++++++ .../test_pg_online_index_durability.py | 203 ++++++++++ tests/unit/test_cell_org_triggers.py | 8 + tests/unit/test_pg_online_index.py | 212 ++++++++++ 10 files changed, 1275 insertions(+), 37 deletions(-) create mode 100644 schemas/router_migrations/004_placements_by_cell_org.sql create mode 100644 server/core/schema/pg_online_index.py create mode 100644 tests/integration/fixtures/pg_migrate_n_minus_one.py create mode 100644 tests/integration/test_pg_online_index.py create mode 100644 tests/integration/test_pg_online_index_durability.py create mode 100644 tests/unit/test_pg_online_index.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..a132706be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -896,10 +896,39 @@ jobs: tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py + tests/integration/test_pg_online_index.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest online-index durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-index-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_PG_INDEX_DRILL: "1" + LOOM_PG_INDEX_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-index-drill-pg loom-scale-index-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-index-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-index-drill-pg loom-scale-index-drill-pg-standby || true + docker network rm loom-scale-index-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby + pytest -q tests/integration/test_pg_online_index_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/development/migrations.md b/docs/development/migrations.md index bbea754e87..0fcc560a45 100644 --- a/docs/development/migrations.md +++ b/docs/development/migrations.md @@ -157,20 +157,56 @@ operator step ([release process](./release-process.md#release-notes)). edited once merged, and every `CREATE`, `ALTER` or `DROP` of a table names `loom.
`, except a rename's new name, where Postgres takes no schema (`tests/unit/test_storage_layout_rows.py`). A boot applies them under a Postgres -advisory lock, each file in one transaction with its ledger row. A migration waits at +advisory lock, each ordinary file in one transaction with its ledger row. A migration waits at most 5 s for a table lock (`lock_timeout`, then retried), so an `ALTER` never queues every query of the table behind a long transaction. A build serves a ledger that holds a later build's files, with a warning naming them; a ledger file it does not carry numbered at or below its own last file, or one whose checksum changed, refuses the boot. -Every file runs in one transaction, so a migration builds no index on a large table: the -build would hold off the table's writes, every commit answering 503, until it ends. Such -an index needs `CREATE INDEX CONCURRENTLY` outside a transaction, and a boot waiting on -the advisory lock deadlocks with it (a waiting session holds a snapshot the build waits -out): a change that adds such a file also takes the lock by retrying -`pg_try_advisory_lock`. When the first release that writes a table carries the file, the -table is small wherever the file runs, and its index may be built in the file's -transaction (`010_revision_delivered_at.sql`). +Ordinary files run in one transaction, so they build no index on a table that +already ships: a transactional build blocks writes until it ends. An index on a +table introduced by the same release may be transactional because that table is +small wherever the file runs (`010_revision_delivered_at.sql`). + +For an existing table, the shared Postgres runner accepts only this online file +form, in either the cell or router journal: + +```sql +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +``` + +The declaration and statement must name the same schema, table, index and ordered +columns. Only ordinary identifiers, a non-unique btree and C collations are +accepted. Expressions, predicates, included columns and additional statements +are refused before the ledger changes. This is not a general escape from +transactional migrations. + +The runner takes session advisory locks by retrying `pg_try_advisory_lock`, with +no transaction or snapshot between attempts. It releases the ordinary migration +key before acquiring a separate schema-scoped online-index key and running +concurrent DDL. A shipped blocking migrator can therefore finish its unchanged +journal while the index waits for old snapshots. After the build finishes, the +runner reacquires the ordinary key, records the file's checksum in the existing +ledger and commits a WAL-message fence. Later files wait for this ledger entry. + +Catalog validation checks the table, ordered columns and collations, btree +method, absence of constraints and expressions, and valid/ready state. A +matching incomplete index is dropped concurrently and rebuilt; a different +same-named object refuses without being dropped. A recorded but absent or +invalid index refuses as corruption. Even when every file is recorded, a build +carrying an online file validates its index and acknowledges a fresh WAL fence: +a primary-visible ledger row after a cancelled synchronous COMMIT is insufficient. +An interrupted build, drop or ledger write is retried from catalogs and the +existing ledger, without a separate job table. Cancellation terminates the +migration session before asyncpg can wait indefinitely on rollback. + +The router's dedicated migration connection has no statement timeout; its +serving pool retains its statement limit. The runner preserves a borrowed +connection's statement budget, including a stopped restore command's deadline. +Concurrent DDL retains the five-second lock timeout and can be interrupted and +retried without changing the file checksum. ## Schema dump and check diff --git a/schemas/router_migrations/004_placements_by_cell_org.sql b/schemas/router_migrations/004_placements_by_cell_org.sql new file mode 100644 index 0000000000..774efeb883 --- /dev/null +++ b/schemas/router_migrations/004_placements_by_cell_org.sql @@ -0,0 +1,3 @@ +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 7876c9f178..ebd230036b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -1,6 +1,7 @@ """Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in -order at boot under a session advisory lock, each file in one transaction with -its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +order at boot under a session advisory lock, each ordinary file in a transaction +with its ledger row (`loom.pg_migrations`, the file's SHA-256). Declared online indexes +run concurrently outside transactions under a separate session key. The router's directory (`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks its session as one that ran the tenancy check and, after the files and before the lock @@ -8,13 +9,14 @@ The rules are the ClickHouse journal's (docs/development/migrations.md): additive only, never edit a merged file. A ledger row whose checksum no longer matches its -file refuses the boot, as does a ledger row for a file this build does not carry -when that file's number is above the build's last one (a newer build's schema is -served, as ClickHouse's is; an unknown lower number means the journal was edited). +file refuses the boot, as does an unknown file numbered at or below this build's +last migration. A build serves a newer schema, warning about unknown files numbered +above its last migration. """ from __future__ import annotations +import asyncio import hashlib import logging from asyncio import sleep @@ -24,7 +26,9 @@ from typing import Any from server.core import pg +from server.core.ch import set_query_tag from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema import pg_online_index from server.core.schema.journal import migration_number, sorted_migrations log = logging.getLogger(__name__) @@ -105,32 +109,126 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files = migration_files(mig_dir) - done: list[str] = [] - # Closing the connection ends the session, and with it the advisory lock. + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") async with (connect or pg.dedicated)() as conn: - # Another boot migrating holds the lock for as long as its files take. - await conn.execute("SELECT pg_advisory_lock($1)", lock_key) - if schema == "loom": - # A migration's data steps only copy or rewrite rows already stored here: the - # org triggers pass them (server/core/cell_triggers.py). - await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") - await conn.execute("SET lock_timeout = '5s'") - await conn.execute(_ledger(schema)) - rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") - for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): - await _apply(conn, name, sql, digest, schema=schema) - log.info("applied Postgres migration %s", name) - done.append(name) - if schema == "loom": - # On every migrating boot, after the pending files and before the lock goes. - from server.core import cell_triggers - - await cell_triggers.ensure(conn) - return done + task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) + try: + return await asyncio.shield(task) + except BaseException: + # Terminate before cancellation can enter asyncpg's unbounded rollback. + conn.terminate() + task.cancel() + await asyncio.wait({task}, timeout=1) + if task.done() and not task.cancelled(): + task.exception() + raise + + +async def _lock(conn: Any, key: int) -> None: + while True: + set_query_tag("migrate.postgres.lock") + if await conn.fetchval("SELECT pg_try_advisory_lock($1)", key): + return + # Each try ends its statement/snapshot before another attempt. + await sleep(0.1) + + +async def _unlock(conn: Any, key: int) -> None: + if not await conn.fetchval("SELECT pg_advisory_unlock($1)", key): + raise RuntimeError("Postgres migration advisory lock was lost") -def _read(files: list[Path]) -> list[tuple[str, str, str]]: - return [(f.name, f.read_text(), checksum(f)) for f in files] +def _online_key(schema: str, lock_key: int) -> int: + value = hashlib.sha256(f"loom.online-index:{schema}:{lock_key}".encode()).digest() + key = int.from_bytes(value[:8], "big") & (2**63 - 1) + if key == lock_key: + raise ValueError("Postgres migration lock keys collide") + return key + + +async def _applied(conn: Any, schema: str, files: list[Path]) -> dict[str, str]: + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + applied = {r["filename"]: r["checksum"] for r in rows} + plan(applied, files) + return applied + + +async def _record_online(conn: Any, path: Path, schema: str, *, pending: bool) -> None: + async with conn.transaction(): + if pending: + await conn.execute( + f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)", + path.name, + checksum(path), + ) + # A visible ledger from a cancelled synchronous commit needs a fresh flush. + await conn.execute(pg._DURABLE_MESSAGE) + + +async def _online( + conn: Any, + path: Path, + index: pg_online_index.OnlineIndex, + schema: str, + lock_key: int, + files: list[Path], +) -> bool: + await _unlock(conn, lock_key) + online_key = _online_key(schema, lock_key) + await _lock(conn, online_key) + applied = await _applied(conn, schema, files) + await pg_online_index.build(conn, index, recorded=path.name in applied) + # A shipped blocking migrator may wait here; the concurrent build is over. + await _lock(conn, lock_key) + applied = await _applied(conn, schema, files) + pending = path.name not in applied + await _record_online(conn, path, schema, pending=pending) + await _unlock(conn, online_key) + return pending + + +async def _migrate( + conn: Any, + files: list[Path], + indexes: dict[str, pg_online_index.OnlineIndex | None], + schema: str, + lock_key: int, +) -> list[str]: + done: list[str] = [] + await _lock(conn, lock_key) + if schema == "loom": + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute("SET synchronous_commit = 'on'") + await conn.execute(_ledger(schema)) + applied = await _applied(conn, schema, files) + for path in files: + set_query_tag("migrate.postgres.apply") + index = indexes[path.name] + if index is not None: + changed = await _online(conn, path, index, schema, lock_key, files) + applied = await _applied(conn, schema, files) + else: + changed = path.name not in applied + if changed: + await _apply(conn, path.name, path.read_text(), checksum(path), schema=schema) + if changed: + log.info("applied Postgres migration %s", path.name) + done.append(path.name) + for index in indexes.values(): + set_query_tag("migrate.postgres.verify") + if index is not None and await pg_online_index.ready(conn, index) is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if schema == "loom": + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + # A caller may lend its marked session and retain unrelated session keys. + await _unlock(conn, lock_key) + return done async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: diff --git a/server/core/schema/pg_online_index.py b/server/core/schema/pg_online_index.py new file mode 100644 index 0000000000..47c2b025a5 --- /dev/null +++ b/server/core/schema/pg_online_index.py @@ -0,0 +1,125 @@ +"""The one nontransactional Postgres migration form: a declared concurrent index.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +_ID = r"[a-z_][a-z0-9_]{0,62}" +_DECLARATION = re.compile( + rf"-- loom:online-index (?P{_ID})\.(?P{_ID}) ON " + rf"(?P{_ID})\.(?P
{_ID}) \((?P{_ID}(?:, {_ID})*)\)\n" +) +_CREATE = re.compile( + rf"CREATE\s+INDEX\s+CONCURRENTLY\s+(?P{_ID})\s+ON\s+" + rf"(?P{_ID})\.(?P
{_ID})\s+USING\s+btree\s*" + rf'\((?P{_ID}\s+COLLATE\s+"C"(?:\s*,\s*{_ID}\s+COLLATE\s+"C")*)\)\s*;\s*' +) + + +@dataclass(frozen=True) +class OnlineIndex: + schema: str + name: str + table: str + columns: tuple[str, ...] + sql: str + + @property + def qualified(self) -> str: + return f'"{self.schema}"."{self.name}"' + + +def parse(sql: str) -> OnlineIndex | None: + """Refuse malformed directives before a connection or ledger write is made.""" + if "loom:online-index" not in sql: + return None + declaration = _DECLARATION.match(sql) + create = _CREATE.fullmatch(sql[declaration.end() :]) if declaration else None + if declaration is None or create is None: + raise ValueError("invalid online-index migration declaration or SQL") + columns = tuple(declaration["columns"].split(", ")) + actual = tuple(part.split()[0] for part in create["columns"].split(",")) + if ( + declaration["schema"] != declaration["table_schema"] + or any(declaration[key] != create[key] for key in ("schema", "name", "table")) + or columns != actual + or len(set(columns)) != len(columns) + ): + raise ValueError("online-index SQL does not match its declaration") + return OnlineIndex( + declaration["schema"], + declaration["name"], + declaration["table"], + columns, + sql[declaration.end() :], + ) + + +_CATALOG = """ +SELECT c.relkind::text AS relkind, tn.nspname AS table_schema, t.relname AS table_name, + am.amname, i.indisvalid, i.indisready, i.indislive, + NOT i.indisunique AND NOT i.indisexclusion AND i.indpred IS NULL + AND i.indexprs IS NULL AND i.indnkeyatts = i.indnatts + AND NOT i.indisreplident AND c.reloptions IS NULL AS plain, + ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.n) AS columns, + ARRAY(SELECT co.collname::text FROM unnest(i.indcollation) WITH ORDINALITY k(oid, n) + JOIN pg_collation co ON co.oid = k.oid + JOIN pg_namespace ns ON ns.oid = co.collnamespace AND ns.nspname = 'pg_catalog' + ORDER BY k.n) AS collations, + ARRAY(SELECT v FROM unnest(i.indoption) v) AS options, + ARRAY(SELECT op.opcdefault FROM unnest(i.indclass) WITH ORDINALITY k(oid, n) + JOIN pg_opclass op ON op.oid = k.oid ORDER BY k.n) AS default_opclasses, + EXISTS(SELECT 1 FROM pg_constraint WHERE conindid = c.oid) + OR EXISTS(SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass + AND objid = c.oid AND deptype IN ('e', 'x')) + OR EXISTS(SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) AS dependent +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_index i ON i.indexrelid = c.oid +LEFT JOIN pg_class t ON t.oid = i.indrelid +LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_am am ON am.oid = c.relam +WHERE n.nspname = $1 AND c.relname = $2 +""" + + +async def ready(conn: Any, index: OnlineIndex) -> bool | None: + """Absent, matching incomplete, or matching ready; foreign objects always refuse.""" + row = await conn.fetchrow(_CATALOG, index.schema, index.name) + if row is None: + return None + expected = { + "relkind": "i", + "table_schema": index.schema, + "table_name": index.table, + "amname": "btree", + "plain": True, + "columns": list(index.columns), + "collations": ["C"] * len(index.columns), + "options": [0] * len(index.columns), + "default_opclasses": [True] * len(index.columns), + "dependent": False, + } + differences = {key: row[key] for key, value in expected.items() if row[key] != value} + if differences: + raise ValueError( + f"online-index {index.qualified} has a different definition or dependency: " + f"{differences}" + ) + return bool(row["indisvalid"] and row["indisready"] and row["indislive"]) + + +async def build(conn: Any, index: OnlineIndex, *, recorded: bool) -> None: + state = await ready(conn, index) + if recorded and state is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if state is True: + return + if state is False: + await conn.execute(f"DROP INDEX CONCURRENTLY {index.qualified}") + await conn.execute(index.sql) + if await ready(conn, index) is not True: + raise ValueError(f"online-index {index.qualified} did not become valid and ready") diff --git a/tests/integration/fixtures/pg_migrate_n_minus_one.py b/tests/integration/fixtures/pg_migrate_n_minus_one.py new file mode 100644 index 0000000000..7876c9f178 --- /dev/null +++ b/tests/integration/fixtures/pg_migrate_n_minus_one.py @@ -0,0 +1,151 @@ +"""Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in +order at boot under a session advisory lock, each file in one transaction with +its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +(`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, +on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks +its session as one that ran the tenancy check and, after the files and before the lock +goes, runs `cell_triggers.ensure` (docs/architecture/cells.md). + +The rules are the ClickHouse journal's (docs/development/migrations.md): additive +only, never edit a merged file. A ledger row whose checksum no longer matches its +file refuses the boot, as does a ledger row for a file this build does not carry +when that file's number is above the build's last one (a newer build's schema is +served, as ClickHouse's is; an unknown lower number means the journal was edited). +""" + +from __future__ import annotations + +import hashlib +import logging +from asyncio import sleep +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Any + +from server.core import pg +from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema.journal import migration_number, sorted_migrations + +log = logging.getLogger(__name__) + +# A fixed key for pg_advisory_lock: every Loom process migrates under it. +_LOCK_KEY = 0x6C6F6F6D_6D696772 # "loommigr" + + +def _ledger(schema: str) -> str: + # `schema` is one of this build's constants (`loom`, `loom_router`), never input. + return f""" +CREATE SCHEMA IF NOT EXISTS {schema}; +CREATE TABLE IF NOT EXISTS {schema}.pg_migrations ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)""" + + +class MigrationDrift(RuntimeError): + """An applied file changed, or the ledger holds a file the journal lost.""" + + +def migration_files(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[Path]: + return sorted_migrations(mig_dir.glob("*.sql")) + + +def checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def plan(applied: dict[str, str], files: list[Path]) -> list[Path]: + """The files still to apply; raises MigrationDrift on a changed or lost file.""" + known = {f.name: f for f in files} + last = max((migration_number(f.name) or 0 for f in files), default=0) + for name, digest in applied.items(): + f = known.get(name) + if f is None: + if (migration_number(name) or 0) <= last: + raise MigrationDrift(f"applied Postgres migration {name} is not in this build") + continue + if checksum(f) != digest: + raise MigrationDrift(f"applied Postgres migration {name} was edited") + newer = sorted(n for n in applied if n not in known) + if newer: + log.warning("Postgres schema has migrations this build does not carry: %s", newer) + return [f for f in files if f.name not in applied] + + +async def pending(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[str]: + """The files `migrate` would apply, read-only: every file when the ledger is not + there yet. For a process that applies none (`LOOM_BOOT_MIGRATE=0`).""" + import asyncpg + + try: + rows = await pg.fetch("SELECT filename, checksum FROM loom.pg_migrations") + except asyncpg.exceptions.UndefinedTableError: + rows = [] + applied = {r["filename"]: r["checksum"] for r in rows} + return [f.name for f in plan(applied, migration_files(mig_dir))] + + +async def migrate( + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + connect: Callable[[], AbstractAsyncContextManager[Any]] | None = None, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply every pending file; returns their names. Concurrent boots serialize on + the advisory lock and the later ones find nothing pending. `connect`, `schema` + and `lock_key` name another database's connection, ledger and lock (the router's + directory); the defaults are the memory authority's. + + On a connection of its own, with no client command timeout and no statement + timeout: DDL on a large table runs as long as it takes. A lock it waits for is + bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never + holds every read and write of the table behind it; the file is retried. For schema + `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files = migration_files(mig_dir) + done: list[str] = [] + # Closing the connection ends the session, and with it the advisory lock. + async with (connect or pg.dedicated)() as conn: + # Another boot migrating holds the lock for as long as its files take. + await conn.execute("SELECT pg_advisory_lock($1)", lock_key) + if schema == "loom": + # A migration's data steps only copy or rewrite rows already stored here: the + # org triggers pass them (server/core/cell_triggers.py). + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute(_ledger(schema)) + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): + await _apply(conn, name, sql, digest, schema=schema) + log.info("applied Postgres migration %s", name) + done.append(name) + if schema == "loom": + # On every migrating boot, after the pending files and before the lock goes. + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + return done + + +def _read(files: list[Path]) -> list[tuple[str, str, str]]: + return [(f.name, f.read_text(), checksum(f)) for f in files] + + +async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: + import asyncpg + + record = f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)" + attempts = 5 # tries before the boot fails on a table lock held past `lock_timeout` + for attempt in range(1, attempts + 1): + try: + async with conn.transaction(): + await conn.execute(sql) + await conn.execute(record, name, digest) + return + except asyncpg.exceptions.LockNotAvailableError: + if attempt == attempts: + raise + log.warning("Postgres migration %s waited too long for a lock; retrying", name) + await sleep(attempt) diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py new file mode 100644 index 0000000000..af93dd0937 --- /dev/null +++ b/tests/integration/test_pg_online_index.py @@ -0,0 +1,373 @@ +"""Online index retries and migrator coexistence on scratch Postgres databases.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate, pg_online_index +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + conn = await asyncpg.connect(dsn, server_settings={"application_name": "online-index-test"}) + try: + yield conn + finally: + await conn.close() + + try: + async with connect() as conn: + yield conn, connect + finally: + await directory.close() + await drop_database(admin, name) + + +def _prefix(tmp_path: Path) -> Path: + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + return prefix + + +async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: + return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) + + +async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None and await pg_online_index.ready(conn, index) is True + assert await conn.fetchval("SELECT to_regclass('loom_router.placements_by_cell') IS NOT NULL") + + +async def test_install_from_empty(database) -> None: + _, connect = database + assert len(await _migrate(connect)) == 4 + + +async def _until(conn: Any, sql: str, *args: Any) -> None: + async with asyncio.timeout(8): + for _ in range(800): + if await conn.fetchval(sql, *args): + return + await asyncio.sleep(0.01) + pytest.fail("Postgres did not reach the expected migration state") + + +def _old_runner(): + """The shipped blocking runner, kept as source for the rolling-upgrade proof.""" + import importlib.util + + path = Path(__file__).parent / "fixtures" / "pg_migrate_n_minus_one.py" + spec = importlib.util.spec_from_file_location("old_pg_migrate", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database, tmp_path) -> None: + conn, connect = database + old_files = _prefix(tmp_path) + await _migrate(connect, old_files) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + first = asyncio.create_task(_migrate(connect)) + second = None + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + second = asyncio.create_task(_migrate(connect)) + # The ordinary key is free while CREATE INDEX waits for this old snapshot. + # Running this original blocking implementation would deadlock if it weren't. + old = _old_runner() + async with asyncio.timeout(2): + assert ( + await old.migrate( + old_files, connect=connect, schema="loom_router", lock_key=8917423 + ) + == [] + ) + assert not first.done() and not second.done() + # Even while the index waits, the old serving path can write and read. + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, host, state, credential_sha256, deployment_id) " + "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" + ) + assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await directory.lookup("org-a") == ("a", "a.example.test") + await tx.rollback() + async with asyncio.timeout(5): + results = await asyncio.gather(first, second) + assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + finally: + if snapshot.is_in_transaction(): + await tx.rollback() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(t for t in (first, second) if t is not None), return_exceptions=True + ) + + +async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs_it( + database, tmp_path +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 2) + assert await conn.fetchval( + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql')" + ) + finally: + await tx.rollback() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + + +@pytest.mark.parametrize( + "definition", + [ + "CREATE TABLE loom_router.placements_by_cell_org (id int)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (org, cell)", + 'CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org COLLATE "POSIX")', + "CREATE UNIQUE INDEX placements_by_cell_org ON loom_router.placements (cell, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WHERE org > 'm'", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) INCLUDE(placed_at)", + "CREATE INDEX placements_by_cell_org ON loom_router.cells (cell, host)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell DESC, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell text_pattern_ops, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WITH (fillfactor=70)", + ], +) +async def test_foreign_same_named_object_is_never_dropped(database, tmp_path, definition) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + await conn.execute(definition) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + with pytest.raises(ValueError, match="different definition"): + await _migrate(connect) + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_recorded_invalid_index_and_checksum_drift_refuse(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute("DROP INDEX loom_router.placements_by_cell_org") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect) + await conn.execute( + "UPDATE loom_router.pg_migrations SET checksum = repeat('0', 64) " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + with pytest.raises(pg_migrate.MigrationDrift): + await _migrate(connect) + + +async def test_cancel_after_catalog_completion_reuses_the_valid_index( + database, tmp_path, monkeypatch +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + original = pg_online_index.build + + async def completed_then_cancel(*args, **kwargs): + await original(*args, **kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr(pg_online_index, "build", completed_then_cancel) + with pytest.raises(asyncio.CancelledError): + await _migrate(connect) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + monkeypatch.setattr(pg_online_index, "build", original) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_keyset_plan_uses_the_ordered_cell_index(database) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute( + "INSERT INTO loom_router.cells (cell, host, state, credential_sha256, deployment_id) " + "SELECT 'cell-' || n, 'cell-' || n || '.example.test', 'serving', " + "md5(n::text) || md5(n::text), 'dep-' || n FROM generate_series(1, 32) n" + ) + await conn.execute( + "INSERT INTO loom_router.placements(org, cell) " + "SELECT 'org-' || lpad(n::text, 5, '0'), 'cell-' || (n % 32 + 1) " + "FROM generate_series(1, 4096) n" + ) + await conn.execute("VACUUM ANALYZE loom_router.placements") + plan = await conn.fetch( + "EXPLAIN SELECT org FROM loom_router.placements " + "WHERE cell = 'cell-2' AND org > 'org-01000' ORDER BY org LIMIT 10" + ) + text = "\n".join(r[0] for r in plan) + assert "placements_by_cell_org" in text and "Sort" not in text + + +async def test_ordinary_files_still_rollback_with_the_ledger(database, tmp_path) -> None: + conn, connect = database + (tmp_path / "001_one.sql").write_text("CREATE TABLE loom_router.one (id int);") + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int); SELECT 1 / 0;") + import asyncpg + + with pytest.raises(asyncpg.DivisionByZeroError): + await _migrate(connect, tmp_path) + assert await conn.fetchval("SELECT to_regclass('loom_router.one') IS NOT NULL") + assert not await conn.fetchval("SELECT to_regclass('loom_router.two') IS NOT NULL") + assert await conn.fetchval("SELECT count(*) FROM loom_router.pg_migrations") == 1 + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int);") + assert await _migrate(connect, tmp_path) == ["002_two.sql"] + + +async def test_invalid_online_declaration_fails_before_the_ledger_exists( + database, tmp_path +) -> None: + conn, connect = database + (tmp_path / "001_bad.sql").write_text("-- loom:online-index bad\nSELECT 1;") + with pytest.raises(ValueError, match="online-index"): + await _migrate(connect, tmp_path) + assert not await conn.fetchval("SELECT to_regclass('loom_router.pg_migrations') IS NOT NULL") + + +async def test_cancelled_concurrent_drop_is_retryable(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None + await conn.execute(index.sql) + async with connect() as reader: + tx = reader.transaction() + await tx.start() + await reader.execute("SET LOCAL enable_seqscan = off") + plan = await reader.fetch( + "EXPLAIN SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org" + ) + assert "placements_by_cell_org" in "\n".join(r[0] for r in plan) + await reader.fetch("SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org") + + async def direct_drop(): + async with connect() as dropping: + try: + await dropping.execute( + "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" + ) + finally: + dropping.terminate() + + dropping = asyncio.create_task(direct_drop()) + try: + query = ( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND query LIKE 'DROP INDEX CONCURRENTLY%' " + "AND wait_event IS NOT NULL)" + ) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + assert await pg_online_index.ready(conn, index) is False + dropping = asyncio.create_task(_migrate(connect)) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + finally: + await tx.rollback() + if not dropping.done(): + dropping.cancel() + await asyncio.gather(dropping, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + + +async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: + conn, connect = database + await conn.execute("SELECT pg_advisory_lock(8917423::bigint)") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid() " + "AND query = 'SELECT pg_try_advisory_lock($1)' " + "AND state = 'idle' AND backend_xmin IS NULL AND xact_start IS NULL)", + ) + assert not migrating.done() + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + assert len(await asyncio.wait_for(migrating, 5)) == 4 + finally: + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +async def test_online_validation_is_repeated_after_later_files(database, tmp_path) -> None: + _, connect = database + for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): + (tmp_path / source.name).write_bytes(source.read_bytes()) + (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py new file mode 100644 index 0000000000..e5644af5af --- /dev/null +++ b/tests/integration/test_pg_online_index_durability.py @@ -0,0 +1,203 @@ +"""A cancelled online-index ledger commit survives a private standby promotion. + +Only its dedicated CI pair enables this destructive drill. Configured failures +are errors; the adopted integration server's replication settings never change. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-index-drill-pg" +STANDBY = PRIMARY + "-standby" +APPLICATION = "online-index-durability" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +async def _waiting(observer, task) -> int: + async with asyncio.timeout(8): + for _ in range(800): + assert not task.done(), "migration acknowledged before standby flush" + pid = await observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + APPLICATION, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.01) + pytest.fail("migration did not reach its synchronous COMMIT") + + +async def _ownership(observer) -> None: + identifier = str(await observer.fetchval("SELECT system_identifier FROM pg_control_system()")) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.recovery +@pytest.mark.destructive +async def test_online_ledger_retry_survives_physical_standby_promotion( + tmp_path, monkeypatch +) -> None: + if not os.environ.get("LOOM_PG_INDEX_DRILL"): + pytest.skip("LOOM_PG_INDEX_DRILL not set: needs its disposable physical standby") + assert os.environ["LOOM_PG_INDEX_DRILL"] == "1" + dsn = os.environ["LOOM_PG_INDEX_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + tasks = [] + paused = False + try: + await _ownership(observer) + _, target = await create_database(dsn) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[asyncpg.Connection]: + conn = await asyncpg.connect( + target, + timeout=5, + server_settings={"application_name": APPLICATION, "synchronous_commit": "on"}, + ) + try: + yield conn + finally: + conn.terminate() + + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + + async def migrate(files: Path = ROUTER_MIGRATIONS_DIR): + return await pg_migrate.migrate( + files, connect=connect, schema="loom_router", lock_key=8917423 + ) + + await migrate(prefix) + record = pg_migrate._record_online + + async def pause_before_ledger(*args, **kwargs): + nonlocal paused + await _docker("pause", STANDBY) + paused = True + await record(*args, **kwargs) + + monkeypatch.setattr(pg_migrate, "_record_online", pause_before_ledger) + first = asyncio.create_task(migrate()) + tasks.append(first) + pid = await _waiting(observer, first) + # The index is acknowledged already; its ledger COMMIT alone is unflushed. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, 3) + async with asyncio.timeout(5): + for _ in range(500): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("cancelled migration retained backend locks") + monkeypatch.setattr(pg_migrate, "_record_online", record) + retry = asyncio.create_task(migrate()) + tasks.append(retry) + await _waiting(observer, retry) + await _docker("unpause", STANDBY) + paused = False + assert await asyncio.wait_for(retry, 8) == [] + observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + database = urlsplit(target).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + database, + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql' AND checksum = '" + + pg_migrate.checksum(ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql") + + "'", + ) + == "1" + ) + assert ( + await _sql( + STANDBY, + database, + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass", + ) + == "t" + ) + finally: + if paused: + await _docker("unpause", STANDBY) + for task in tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + observer.terminate() diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 769cfe1bb3..5d25ba0177 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -86,6 +86,14 @@ async def execute(self, sql: str, *_a: Any) -> None: async def fetch(self, _sql: str, *_a: Any) -> list[Any]: return [] + def terminate(self) -> None: + self.statements.append("terminated") + + async def fetchval(self, sql: str, *_a: Any) -> bool: + assert sql in ("SELECT pg_try_advisory_lock($1)", "SELECT pg_advisory_unlock($1)") + self.statements.append(sql) + return True + @pytest.mark.parametrize("name", ["eu-1", ""]) @pytest.mark.parametrize("schema", ["loom", "loom_router"]) diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py new file mode 100644 index 0000000000..7fc43fa2bb --- /dev/null +++ b/tests/unit/test_pg_online_index.py @@ -0,0 +1,212 @@ +"""Only the declared concurrent btree form escapes a migration transaction.""" + +from __future__ import annotations + +import pytest + +from server.core.schema import pg_online_index + +SQL = """-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +""" + + +def test_the_online_index_declaration_has_one_exact_target() -> None: + parsed = pg_online_index.parse(SQL) + assert parsed is not None + assert parsed.schema == "loom_router" + assert parsed.name == "placements_by_cell_org" + assert parsed.table == "placements" + assert parsed.columns == ("cell", "org") + assert pg_online_index.parse("CREATE TABLE loom.t (id int);") is None + + +@pytest.mark.parametrize( + "sql", + [ + SQL + "SELECT 1;", + SQL.replace("CREATE INDEX", "CREATE UNIQUE INDEX"), + SQL.replace('org COLLATE "C"', 'org COLLATE "POSIX"'), + SQL.replace("(cell, org)", "(org, cell)"), + SQL.replace("ON loom_router.placements\n", "ON loom_router.other\n"), + SQL.replace("USING btree", "USING hash"), + SQL.replace('org COLLATE "C"', 'lower(org) COLLATE "C"'), + SQL.replace(");", ") WHERE true;"), + SQL.replace("-- loom:online-index", "-- loom:online-index-bad"), + SQL.replace("placements_by_cell_org ON", "other_index ON", 1), + SQL.replace("(cell, org)", "(cell, cell)"), + SQL.replace("CONCURRENTLY", ""), + ], +) +def test_other_online_sql_is_refused(sql: str) -> None: + with pytest.raises(ValueError, match="online.index"): + pg_online_index.parse(sql) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pending", [True, False]) +async def test_ledger_retries_commit_a_wal_fence_even_when_already_recorded( + tmp_path, pending +) -> None: + import contextlib + + from server.core import pg + from server.core.schema import pg_migrate + + path = tmp_path / "001_index.sql" + path.write_text(SQL) + + class Connection: + def __init__(self): + self.statements = [] + + def transaction(self): + return contextlib.nullcontext() + + async def execute(self, sql, *args): + self.statements.append((sql, args)) + + conn = Connection() + await pg_migrate._record_online(conn, path, "loom_router", pending=pending) + assert conn.statements[-1] == (pg._DURABLE_MESSAGE, ()) + assert len(conn.statements) == (2 if pending else 1) + if pending: + assert conn.statements[0][1] == (path.name, pg_migrate.checksum(path)) + + +@pytest.mark.asyncio +async def test_migrator_preserves_a_borrowed_sessions_statement_budget(tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Connection: + def __init__(self): + self.statements = [] + + async def execute(self, sql, *args): + self.statements.append(sql) + + async def fetch(self, sql): + return [] + + async def fetchval(self, sql, *args): + self.statements.append(sql) + return True + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + assert await pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") == [] + assert not any("statement_timeout" in sql for sql in conn.statements) + assert "SET synchronous_commit = 'on'" in conn.statements + assert conn.statements[-1] == "SELECT pg_advisory_unlock($1)" + + +@pytest.mark.asyncio +async def test_online_target_must_match_its_ledger_before_connecting(tmp_path) -> None: + from server.core.schema import pg_migrate + + (tmp_path / "001_index.sql").write_text(SQL) + + def never_connect(): + pytest.fail("invalid target opened a connection") + + with pytest.raises(ValueError, match="ledger schema"): + await pg_migrate.migrate(tmp_path, schema="loom", connect=never_connect) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) +async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( + tmp_path, phase +) -> None: + import asyncio + import contextlib + from types import SimpleNamespace + + from asyncpg.transaction import Transaction + + from server.core.schema import pg_migrate + + (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") + reached = asyncio.Event() + terminated = asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def is_closed(self): + return terminated.is_set() + + def transaction(self): + return Transaction(self, None, False, False) + + def terminate(self): + terminated.set() + + async def fetchval(self, sql, *args): + return True + + async def fetch(self, sql): + return [] + + async def execute(self, sql, *args): + if sql == "SELECT stalled_statement;" and phase == "rollback": + raise ValueError("statement failed before rollback") + if ( + (sql == "SELECT stalled_statement;" and phase == "statement") + or (sql.startswith("ROLLBACK") and phase == "rollback") + or (sql.startswith("COMMIT") and phase == "commit") + ): + reached.set() + # Transport termination, rather than a second task cancellation, + # releases a transaction that has already begun async cleanup. + while not terminated.is_set(): + try: + await terminated.wait() + except asyncio.CancelledError: + continue + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + migrating = asyncio.create_task( + pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") + ) + try: + await asyncio.wait_for(reached.wait(), 1) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 1) + assert terminated.is_set() + finally: + terminated.set() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +def test_postgres_journals_keep_distinct_numbers_and_declared_online_forms() -> None: + from server.core.paths import PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR + from server.core.schema import pg_migrate + from server.core.schema.journal import migration_number + + for directory in (PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR): + files = pg_migrate.migration_files(directory) + numbers = [migration_number(path.name) for path in files] + assert None not in numbers and len(numbers) == len(set(numbers)) + for path in files: + sql = path.read_text() + parsed = pg_online_index.parse(sql) + if "CONCURRENTLY" in sql: + assert parsed is not None, f"{path.name} lacks an online-index declaration" From cc4ed54549256b07e35101b1bdc17d68e050cb5b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:09:48 +0000 Subject: [PATCH 037/181] Fence cell control mutations across local lock loss Claim a fresh directory UUID before acceptance writes, and compare the generation, registration and exact fingerprint pair on control mutations. Keep every local acceptance transaction on its attach-lock backend and stop after a lost session. The additive router column permits older data-plane reads and inserts. Online control requires turnover of older clients and router handlers; the recovery guide covers claimed state cleanup and offline generation rotation. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 61 ++ docs/operations/cell-control.md | 138 ++++ docs/operations/troubleshooting.md | 12 +- .../005_control_generation.sql | 3 + server/cli/cell.py | 119 ++-- server/cli/cell_attach.py | 126 ++-- server/cli/cell_attach_pg.py | 104 +-- server/cli/cell_control.py | 246 +++++++ server/cli/cell_prepare.py | 85 +-- server/router/app.py | 123 +++- server/router/directory.py | 153 +++-- tests/integration/test_cell_attach.py | 11 +- tests/integration/test_cell_control_fence.py | 613 ++++++++++++++++++ tests/integration/test_router_directory_pg.py | 179 +++-- tests/unit/test_cell_attach.py | 133 ++-- tests/unit/test_cell_control.py | 123 ++++ tests/unit/test_cell_operator.py | 45 +- tests/unit/test_cell_prepare.py | 33 +- tests/unit/test_cell_register.py | 20 +- tests/unit/test_router.py | 12 +- tests/unit/test_router_attach.py | 209 ++++-- 21 files changed, 2054 insertions(+), 494 deletions(-) create mode 100644 docs/operations/cell-control.md create mode 100644 schemas/router_migrations/005_control_generation.sql create mode 100644 server/cli/cell_control.py create mode 100644 tests/integration/test_cell_control_fence.py create mode 100644 tests/unit/test_cell_control.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index dd1189b696..e5419326ed 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1205,6 +1205,67 @@ which writes the directory alone, with the router's operator credential and none cell's stores, so it works while the cell is down; the router judges a placement from the directory row. +## Control mutation fencing + +A cell controller holds the session-level attach lock on one direct, marked, UTF8 +Postgres connection for its whole run. Every write to `loom.cell_self`, `loom.cell_first` +or `loom.tenancy_mode` checks that lock and its observed local state, then writes and +commits on that exact connection. Prepare passes its lock connection to its row writer; +attach commits mode on its lock connection. A lost connection ends the run. It cannot +reconnect and continue an earlier claim. Terminating the client does not prove its +server backend ended: another controller must actually acquire the attach lock before +it starts, and an unresponsive backend can keep that lock until it is positively ended. + +The directory stores one opaque UUID, `cells.mutation_generation`, for each cell. +`POST /cells/v1/operator/cells//claim` takes the directory advisory lock and cell +row lock, compares the full observed registration and exact current/next JSON pair, +generates a fresh UUID on the server, audits it and commits synchronously before +answering. Every execution generates another UUID, including a repeated HTTP request. +The caller never proposes a replacement UUID. Registration assigns a fresh default; +a registration rerun reads the existing value without resetting it. Removing and +registering the same name therefore cannot restore an earlier generation. + +Before its first local acceptance write, prepare, attach or a recovery controller +claims the generation while holding its local lock. Every subsequent directory +mutation carries `expected`: cell, first-cell flag, host, deployment id, bucket, prefix, +probe/upstream addresses, exact current/next pair and the claimed generation. The claim +carries the same fields without a generation. The directory compares them under its +locks before any mutation or no-op success. Canonical JSON comparison distinguishes +`false`, `0` and `0.0`. A stale request cannot release a reservation after its local +backend dies and another controller claims and widens acceptance, even if the directory +pair later returns to its earlier values. A delayed claim can invalidate another +controller's work; it cannot let a dead local session write acceptance state. + +A lost acknowledgement is an unknown outcome. Stop the attempt and rerun: obtain the +local lock, read current state and claim a fresh UUID before recomputing the action. +Do not copy a latest UUID onto an old action. A successful claim or control transaction +waits through synchronous COMMIT. A no-op transaction emits transactional WAL so it +also proves an observed earlier write durable. Acquisition, statements, COMMIT and +cleanup have explicit bounds; an interrupted or uncertain local transaction never +continues on another connection. No operation record, persistent census, phase or lease +is added. The UUID is coordination metadata and holds no memory content. + +The source census for acceptance and operator directory writes is: + +| Writer | Fence | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | +| `cell_attach_pg._mode_try` | Attach's claim, recording/first/fingerprint comparison, exact lock backend | +| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | +| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | +| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | +| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | + +Read-only routes are exempt. Router migrations and an offline directory restore are +maintenance operations. Future restore/update acceptance writers must use the same +claim and lock-session helpers; a new directory mutation must use the common comparison +before its no-op branch as well. Direct privileged SQL is not an online control writer. +The [control upgrade and recovery procedure](../operations/cell-control.md) defines the +required fleet turnover and offline restore of generations. N-1 data-plane builds can +read and insert on the additive schema; old control clients and router handlers cannot +participate in this protocol. + ## Operations - `loom cell restore`, after any store restore and before `loom reconcile`, writes the diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md new file mode 100644 index 0000000000..ccd7bf5edf --- /dev/null +++ b/docs/operations/cell-control.md @@ -0,0 +1,138 @@ +# Cell control upgrades and recovery + +Cell control mutations require generation-aware clients and router handlers, and direct +access to the cell's Postgres. This page covers the runtime prerequisite, unknown +outcomes and registered-cell state cleanup. The protocol is described in +[cells](../architecture/cells.md#control-mutation-fencing). + +## Upgrade the control fleet + +Apply the router migrations with `loom router migrate`. The additive UUID column has a +server default and remains readable by N-1 data-plane code. Its volatile default may +rewrite the small cells table and takes a table lock; allow a bounded maintenance window. +Schema compatibility does not let old control programs participate in fencing. + +Before using generation-aware acceptance mutation, inventory and positively terminate +all old router control handlers and prepare, attach, restore, update, remove and manual +cleanup clients, including paused one-offs and outstanding requests. Replace every +launcher/image so those binaries cannot restart against the stores. Use exact process +and start identities, trustworthy evidence that the exact container ended, or independent +node fencing. A heartbeat expiry, missing pod object or one successful request to a router +replica does not prove this inventory. If the inventory is incomplete, stop/fence every +machine that could still run an old control program before resuming control traffic. +Compatible cell data-plane builds may continue serving during this procedure. Every +old router replica that could execute an operator handler must be positively retired; +continuing its data-plane routing alone does not remove that control capability. + +The fencing changes and their dependent CLI changes may be implemented and reviewed as +stacked concerns, merged in dependency order and shipped together. They require no +separate release cut. Running a protected operation requires the capable runtime and +completed control turnover procedure. Missing-generation requests refuse with +`CONTROL_UPGRADE_REQUIRED`; an absent claim route or incomplete operator snapshot tells +the CLI to upgrade the router replica. There is no compatibility fallback or operator +flag that proves a deployment completed its external process inventory. + +## Unknown outcomes + +A timeout, connection loss, cancelled COMMIT or lost acknowledgement ends the current +attempt. Keep the exact cell and router configuration, inspect the reported state and +rerun the command. The rerun must acquire the local attach lock and make a fresh claim +before it changes acceptance. A duplicate claim creates a fresh UUID; it never restores +a caller's earlier token. Do not replay a saved mutation with a copied current token. + +A client connection ending does not prove its Postgres backend ended. If the attach lock +remains held, inspect its exact holder with the query in +[troubleshooting](troubleshooting.md#a-process-with-loom_cell-unset-is-gated). Positively +end that exact owned backend, or wait until it ends. Do not terminate whichever backend +holds a mode-table lock: it may belong to another controller. No control run reconnects +to continue an uncertain attempt. + +## Clear a registered unattached cell's state + +Use this only for the troubleshooting page's eligible unattached cell with no placements, +or a first cell being backed out while its local mode is still `recording`. A first cell +in local `enforced` mode or any cell with placements goes forward. Complete the control +fleet turnover above, retain the existing stop/unset-`LOOM_CELL` procedure, and prevent +accepting or maintenance processes from restarting with the cell setting during cleanup. +Use the table-owning migration role. Never clear a restored copy beside its live source. + +Run the following with this cell's settings, including `LOOM_CELL`, `LOOM_PG_DSN`, its +ClickHouse and bucket configuration, `LOOM_CELL_ROUTER_URL` and the operator credential. +This read-only store pairing check must succeed before the helper claims or clears state. +It does not write participant rows or run the data-plane command gate. + +```python +import argparse +import asyncio + +import httpx + +from server.cli import cell as cell_cli +from server.cli import cell_control +from server.core import pg, pg_profile +from server.core.ch import get_client +from server.settings import settings + + +async def main(): + ch = get_client() + await ch.connect() + try: + recorded = await pg_profile.recorded_authority(ch) + if not recorded or recorded["authority"] != "postgres": + raise RuntimeError("the configured stores do not record Postgres authority") + pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) + url = cell_cli.operator_router(argparse.Namespace(router=None)) + if not isinstance(url, httpx.URL): + raise RuntimeError("the router URL or operator credential was refused") + + async def call(path, body): + return await cell_control.request(url, settings.cell, None, path, body) + + async with cell_control.session("state cleanup") as conn: + row = await cell_control.request(url, settings.cell, None, "", method="GET") + await cell_control.backout(conn, call, row) + finally: + await ch.disconnect() + + +asyncio.run(main()) +``` + +The helper claims first, closes the cell using the existing limit operation, and requires +that operation's durable placement count to remain zero before any local state cleanup. +A queued placement either wins first and refuses cleanup, or observes the closed row and +cannot create a placement. The helper then compares and deletes only the existing +`cell_first`, `tenancy_mode`, `cell_self` and `tenancy` rows, restores the first-row +triggers and drops only the existing org guard triggers, on the same locked backend. +It takes the migration lock for each bounded local transaction. Even an already-open +retry proves durability before reporting success. + +An abort leaves the cell closed. It never reopens automatically or deletes a placement. +If a placement won, resume attach; inspect the reason before explicitly reopening with +`loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run +`loom cell remove --cell ` with the same cell settings. Removal now holds that +cell's direct Postgres lock and takes a fresh claim for both first and later cells. It +also requires a first cell's durable local state to read `open` before deleting the row. +Only then may the deployment be prepared again. + +## Restore the directory offline + +A directory backup can contain an earlier generation. Stop and positively terminate or +fence all control clients and router processes, including outstanding control requests, +and prevent restarts before restoring it. While control traffic remains stopped, execute +this on the restored directory and wait for its synchronous commit: + +```sql +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '10s'; +UPDATE loom_router.cells SET mutation_generation = gen_random_uuid(); +COMMIT; +``` + +Bound the client's complete transaction/COMMIT wait too. An uncertain outcome does not +permit APIs to resume: establish the restored writer's actual completion, retry the +rotation while still offline and obtain a known durable commit. Rotate every restored +row before restarting capable router/control binaries. Never copy generations from a +cell backup or rely on an audit retention window to fence restored requests. diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 3ede5f4702..1ca3ce0a40 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -295,7 +295,12 @@ each field means is in [monitoring](./monitoring.md#health-checks). - **The deployment is not meant to be a cell** (no directory row, or no router), or **an unattached later cell** (its row reads `is_first = false`, `state = 'attaching'` and `placed = 0`) that you are turning back into a single - deployment: clear the state in one transaction, in a psql session on `LOOM_PG_DSN` + deployment: for a registered cell, use the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state). + Do not run the SQL below for a registered cell. For a deployment whose directory + has no row, first positively stop/fence every control client and outstanding router + control request and prevent restarts; with no online acceptance controller, clear + the state in one transaction, in a psql session on `LOOM_PG_DSN` (`\connect` back to it after the check) as the role that owns schema `loom`'s tables (the role that migrates; altering a trigger needs the table's owner). The deployment is then `open`. An unattached later cell's directory row stays live until @@ -353,8 +358,9 @@ each field means is in [monitoring](./monitoring.md#health-checks). `state = 'attaching'` and `placed = 0`, whatever its `mode`, since an attach refused at its step-3 lock leaves it `enforced`; and `SELECT mode FROM loom.tenancy_mode` on its Postgres reads `recording`): back it out in this order. Unset `LOOM_CELL` on every - process, each of which then serves as in `recording`; run the state transaction and the - drop step above; then run `loom cell remove --cell ` with the router's operator + process, each of which then serves as in `recording`; run the + [fenced state cleanup](cell-control.md#clear-a-registered-unattached-cells-state), + with `LOOM_CELL` still set on that control client; then run `loom cell remove --cell ` with the router's operator credential and the cell's `LOOM_PG_DSN`, which refuses until that Postgres reads `open` and records the row's deployment id. Removing the row first would answer every new org 503, and clearing the state first would gate every process with `LOOM_CELL` set. The diff --git a/schemas/router_migrations/005_control_generation.sql b/schemas/router_migrations/005_control_generation.sql new file mode 100644 index 0000000000..82baab882f --- /dev/null +++ b/schemas/router_migrations/005_control_generation.sql @@ -0,0 +1,3 @@ +-- A cell control claim invalidates requests from every earlier local lock owner. +-- Fresh values on insertion and each claim also fence removal/re-registration. +ALTER TABLE loom_router.cells ADD COLUMN mutation_generation uuid NOT NULL DEFAULT gen_random_uuid(); diff --git a/server/cli/cell.py b/server/cli/cell.py index 65f18990d8..6d6f23697f 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -1,6 +1,7 @@ """`loom cell credential | limit | which | remove`: the operator's commands that touch the -router's directory (docs/architecture/cells.md, "Operations"). Only `remove` reads a cell's -stores, a first cell's Postgres; the others run while a cell is down. `loom cell prepare` is +router's directory (docs/architecture/cells.md, "Operations"). Removal holds the cell's +Postgres lock and claims its directory generation; limits and credentials need only the +directory. `loom cell prepare` is `server/cli/cell_prepare.py`, `loom cell attach` `server/cli/cell_attach.py`. - `credential new` makes a router credential offline and prints it with its SHA-256; @@ -10,7 +11,8 @@ - `which --org ` prints the cell and host the router's lookup answers for an org (the first cell's for an org with no placement, which it cannot tell apart). - `remove --cell ` removes the directory row of a cell that is `attaching` with no - placement; for the first cell, only once its Postgres reads `open` and records the row's + placement, holding its configured Postgres lock; for the first cell, only once it reads + `open` and records the row's deployment id (the troubleshooting page's back-out runs first). `credential` (except `new`), `limit` and `remove` call the router's operator API with @@ -33,6 +35,7 @@ from server.auth.public_url import is_loopback_host from server.core.cell import NAME, PLACEMENTS_MAX, SHA256, TROUBLESHOOTING from server.router.directory import ( + CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S, @@ -45,7 +48,7 @@ # A lookup makes up to two directory reads (the org, then the first cell), each a # slot-and-connection wait and a statement. A connect fails fast. _OPERATOR_TIMEOUT = httpx.Timeout( # not a setting: derived - max(OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 + max(CONTROL_MAX_S, OPERATOR_CHANGE_MAX_S, OPERATOR_READ_MAX_S, REMOVE_MAX_S) + 10.0, connect=5.0 ) _LOOKUP_TIMEOUT = httpx.Timeout( # not a setting: derived 2 * (2 * STATEMENT_TIMEOUT_S + 1) + 10.0, connect=5.0 @@ -155,6 +158,18 @@ def _operator_call( if isinstance(found, int): return found router = found + if path in ("limit", "credential"): + from server.cli import cell_control + + row = _read_row(router, args.cell) + if isinstance(row, str): + print(f"loom cell: {row}", file=sys.stderr) + return 1 + try: + body = {**(body or {}), **cell_control.preconditions(row)} + except cell_control.Refused as e: + print(str(e), file=sys.stderr) + return 1 try: r = _send(router, method, args.cell, path, body) except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ProxyError) as e: @@ -306,43 +321,6 @@ def _read_row(router: httpx.URL, cell: str) -> dict[str, Any] | str: return answer -# The first cell's check: one Postgres session, its statements and the whole read bounded. -_CHECK_STATEMENT_MS = 5000 # not a setting: a bound on one CLI read, not on serving -_CHECK_S = 10.0 # not a setting: a bound on one CLI read, not on serving - - -async def _read_first_cell() -> tuple[str, str]: - """The cell state and the deployment id the cell's Postgres records ('' when none), on - one session of its own: no ClickHouse and no pool.""" - from server.core import cell, pg - - async with pg.dedicated(statement_timeout_ms=_CHECK_STATEMENT_MS) as conn: - state, _first = await cell.read_state_if_present(conn) - recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") - return state, str(recorded or "") - - -async def _first_cell_refusal(deployment_id: str) -> str: - """Why the first cell's row may not be removed yet ('' when it may): its Postgres still - holds a cell state, or is another deployment's. It reads `LOOM_PG_DSN` alone.""" - if not settings.pg_dsn: - return "the first cell keeps its state in Postgres: set LOOM_PG_DSN to the cell's" - state, recorded = await asyncio.wait_for(_read_first_cell(), timeout=_CHECK_S) - if state != "open": - return ( - f"the first cell's Postgres reads `{state}`: unset LOOM_CELL on every process, run " - f"the full-state cleanup ({TROUBLESHOOTING}), then remove the row" - ) - if not recorded: - return "this Postgres records no deployment id: point LOOM_PG_DSN at the cell's own" - if recorded != deployment_id: - return ( - f"this Postgres records deployment id {recorded!r}, and the directory's row " - f"{deployment_id!r}: point LOOM_PG_DSN at the cell's own Postgres" - ) - return "" - - def _cmd_remove(args: argparse.Namespace) -> int: router = _cell_router(args) if isinstance(router, int): @@ -370,24 +348,47 @@ def _cmd_remove(args: argparse.Namespace) -> int: if reason := remove_refusal(args.cell, state, placed): print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 - if row["is_first"]: - try: - refusal = asyncio.run(_first_cell_refusal(row["deployment_id"])) - except Exception as e: # noqa: BLE001 — an unread state refuses the removal - reason = f"no answer within {_CHECK_S:g} s" if isinstance(e, TimeoutError) else str(e) - refusal = f"the first cell's Postgres could not be read ({type(e).__name__}: {reason})" - if refusal: - print(f"loom cell remove: refusing: {refusal}", file=sys.stderr) - return 1 - return _operator_call( - args, - "", - None, - "a retry refused as not in the directory (NO_SUCH_CELL) means the row was removed", - method="DELETE", - router=router, - coded_404=True, - ) + try: + answer = asyncio.run(_remove_locked(args, router, row)) + except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run + print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + +async def _remove_locked( + args: argparse.Namespace, + router: httpx.URL, + row: dict[str, Any], + transport: httpx.AsyncBaseTransport | None = None, +) -> dict[str, Any]: + from server.cli import cell_control + from server.core import cell + + async with cell_control.session("remove") as conn: + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + # Re-read under the local lock; the earlier read only checked eligibility. + row = await cell_control.request(router, args.cell, transport, "", method="GET") + if reason := remove_refusal(args.cell, row["state"], row["placed_orgs"]): + raise cell_control._refused(reason) + await cell_control.paired(conn, row) + if row["is_first"]: + async with cell_control.transaction(conn): + state, _ = await cell.read_state_if_present(conn) + if state != "open": + raise cell_control._refused( + f"the first cell's Postgres reads {state}: run the full-state cleanup " + f"({TROUBLESHOOTING}) before removing its directory row" + ) + row = await cell_control.claim(conn, call, row) + await cell_control.check_lock(conn) + return await cell_control.request( + router, args.cell, transport, "", cell_control.preconditions(row), method="DELETE" + ) def _cmd_which(args: argparse.Namespace) -> int: diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index bf4e81521e..b3cb44d9cf 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -42,7 +42,7 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli import cell_scan +from server.cli import cell_control, cell_scan from server.cli._ch import writes from server.cli.cell_ch import Refusal, Refused, read_topology from server.core import cell, cell_triggers @@ -77,32 +77,15 @@ def __init__( ) -> None: self.url, self.name, self.transport = url, name, transport - def _client(self) -> httpx.AsyncClient: - return httpx.AsyncClient( - timeout=cell_cli._OPERATOR_TIMEOUT, - trust_env=self.url.scheme == "https", - transport=self.transport, - headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) - async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: - """The router's 200 answer naming the cell; Refused with its refusal otherwise.""" - url = cell_cli.at(self.url, f"/cells/v1/operator/cells/{self.name}" + path) - try: - async with self._client() as client: - r = await (client.get(url) if body is None else client.post(url, json=body)) - except httpx.HTTPError as e: - raise _refused("router", f"no answer ({type(e).__name__}): rerun attach") from None - answer = cell_cli.router_answer(r, {"cell": self.name}, cell_cli.OPERATOR_REFUSALS) - if answer is None: - raise _refused("router", cell_cli.not_the_router(r)) - if r.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": - raise _refused("router", f"a router replica is not upgraded: it does not answer {url}") - if r.status_code != 200: - raise _refused( - "router", f"refused {path or 'the read'} ({r.status_code}): {answer['error']}" - ) - return answer + return await cell_control.request( + self.url, + self.name, + self.transport, + path, + body, + method="GET" if body is None else "POST", + ) async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" @@ -183,13 +166,8 @@ def __init__( # -- the lock session -------------------------------------------------------------- async def open_lock(self) -> None: - from server.core import pg - - self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + self._lock_cm = cell_control.session("attach") self.lock_conn = await self._lock_cm.__aenter__() - if refusals := await apg.direct_refusals(self.lock_conn, "attach"): - raise Refused(refusals) - await apg.take_lock(self.lock_conn, "attach") async def close_lock(self) -> None: if self._lock_cm is not None: @@ -197,33 +175,14 @@ async def close_lock(self) -> None: self._lock_cm = self.lock_conn = None async def confirm_lock(self) -> None: - """The lock is held still, or taken again (refusing when another run holds it now), - and the cell's state read again: a cleanup, a prepare rerun or another attach could - have run in the gap.""" - try: - if await apg.lock_held(self.lock_conn): - return - except Exception as e: # noqa: BLE001 — a lost session is taken again below - self.log(f"the lock session was lost ({type(e).__name__}); taking the lock again") - else: - self.log("the attach lock was lost; taking it again") - try: - await self.close_lock() - except Exception: # noqa: BLE001 — the old session is gone either way - self._lock_cm = self.lock_conn = None - await self.open_lock() - state, first = await apg.read_mode(self.lock_conn) - if first != (True, self.name) or (self.mode == "enforced" and state != "enforced"): - raise _refused( - "cell state", - f"changed while the lock was lost (now {state}, first row {first}): rerun attach", - ) - row = await self.router.call("") - if ( - row.get("deployment_id") != self.row.get("deployment_id") - or row.get("state") != "attaching" - ): - raise _refused("directory row", "changed while the lock was lost: rerun attach") + """A lost backend ends the owner; a new run must claim and start again.""" + await cell_control.check_lock(self.lock_conn) + + async def mutate(self, path: str, body: dict[str, Any]) -> dict[str, Any]: + answer = await cell_control.mutate(self.lock_conn, self.router.call, self.row, path, body) + if "mutation_generation" in answer: + self.row = {**self.row, **answer} + return answer # -- the refusals ------------------------------------------------------------------ @@ -406,6 +365,7 @@ async def begin(self) -> None: "pg_allow_roles": sorted(self.args.pg_allow_role), "nonces": nonces, } + body.update(cell_control.preconditions(self.row)) if (size := _encoded_size(body)) > ATTACH_BEGIN_MAX_BYTES: raise _refused( "attach/begin", @@ -414,7 +374,7 @@ async def begin(self) -> None: ) try: await _write_nonces(self.ch, nonces) - self.row = {**self.row, **await self.router.call("/attach/begin", body)} + self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp finally: await _delete_nonces(self.ch, nonces) @@ -451,28 +411,21 @@ async def ready_for_commit(self) -> list[str]: return verdict.booting async def commit_mode(self) -> None: - from server.core import pg - - await self.router.call("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) def mark(mode: str) -> None: - # `unknown` from a try's UPDATE on: only the durable read can say whether it - # landed; `recording` again when that try rolled back. self.mode = mode - try: - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark - ) - except Refused: - raise - except Exception as e: - if self.mode != "unknown": - raise # no try was in flight: nothing to decide - self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.end_mode_transactions(conn, self.confirm_lock) + await apg.commit_enforced( + self.lock_conn, + self.confirm_lock, + self.ready_for_commit, + self.wait, + self.log, + mark, + expected_cell=self.name, + expected_fp=self.own_fp, + ) async def durable(self) -> int: """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's @@ -484,13 +437,14 @@ async def durable(self) -> int: if await pg_profile.standby_now() != "synchronous": raise _refused("postgres standbys", "not synchronous before the durable read") try: - read = await apg.durable_read() + read = await apg.durable_read(self.lock_conn) except Exception as e: # A try cut off by its bound, or an outage: its outcome is unknown, and # the next try reads again. Anything else is no reason to retry. outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) if not (isinstance(e, TimeoutError) or outage): raise + await self.confirm_lock() self.log(f"the durable read failed ({type(e).__name__}); retrying") if not await self.wait.pause(): raise _refused( @@ -573,7 +527,7 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: placed = 0 for batch in batches(orgs, self.row["deployment_id"]): await self.confirm_lock() - answer = await self.router.call( + answer = await self.mutate( "/attach/place", {"deployment_id": self.row["deployment_id"], "orgs": batch} ) placed += int(answer.get("placed") or 0) @@ -586,7 +540,9 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: await self.final_checks(watch) row = await self.router.call("") current, nxt = self.own_fp - if (row.get("current_fp"), row.get("next_fp")) != (current, nxt): + if json.dumps((row.get("current_fp"), row.get("next_fp")), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): raise _refused( "postgres loom.cell_self", "differs from the directory's fingerprints (a fingerprinted setting changed " @@ -602,7 +558,7 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.router.call( + return await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) finally: @@ -627,6 +583,7 @@ async def done(self) -> bool: async def run(self) -> int: self.row = await self.router.call("") + cell_control.preconditions(self.row) if self.row.get("state") == "serving": await self.done() self.report.update(state="serving", done=True) @@ -637,6 +594,7 @@ async def run(self) -> int: topology, names, listed = await self.refusals() if self.args.dry_run: return await self.dry_run(topology, listed) + self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) await self.begin() state, _ = await apg.read_mode(self.lock_conn) self.mode = state @@ -651,9 +609,7 @@ async def run(self) -> int: await self.precommit(found) await self.commit_mode() else: - await self.router.call( - "/attach/enforced", {"deployment_id": self.row["deployment_id"]} - ) + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) xmax = await self.durable() await self.check_beats("ignore") await self.step4(xmax, topology, names) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 12e1808f8b..8ae6436f3f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -11,6 +11,7 @@ import asyncio import contextlib +import json import time from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass @@ -581,21 +582,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: # The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on # loom.tenancy_mode in this database, which only attach takes. -_MODE_HOLDERS = """ -SELECT l.pid, a.application_name, a.xact_start, r.rolsuper, l.granted -FROM pg_locks l -JOIN pg_stat_activity a ON a.pid = l.pid -LEFT JOIN pg_roles r ON r.oid = a.usesysid -WHERE l.locktype = 'relation' AND l.relation = 'loom.tenancy_mode'::regclass - AND l.mode = 'ExclusiveLock' - AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) - AND l.pid <> pg_backend_pid() -""" - _TIMED_OUT = frozenset({"55P03", "57014"}) -# How long attach waits for an ended mode transaction's lock to clear. -_END_ROUNDS = 30 # not a setting: checks of the lock -_END_PAUSE_S = 0.1 # not a setting: the pause between them async def commit_enforced( @@ -605,22 +592,20 @@ async def commit_enforced( wait: Wait, log: Callable[[str], None], mark: Callable[[str], None], + *, + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], ) -> None: - """Raise the mode to `enforced` on `conn` (a session of its own, never the lock - session): an EXCLUSIVE lock on loom.tenancy_mode a try, `_MODE_LOCK_MS` each (less - when `wait` has less left), within `wait`; `confirm_lock` before each try and each - termination (it refuses when another run holds the attach lock), and `ready` before - each try (the booting process ids skip the try). After a failed try the in-flight - mode transaction, found by its lock, is ended. `mark` hears `unknown` just before a try - sends its UPDATE and `recording` when that try rolled back. Raises Refused when `wait` - runs out; whether the commit landed is the durable read's to say.""" + """Raise mode on the exact ATTACH-lock backend; an unknown COMMIT ends this run.""" with wait.counting(): while True: await confirm_lock() booting = await ready() if not booting: try: - done = await _mode_try(conn, wait.bound_ms(_MODE_LOCK_MS), mark) + done = await _mode_try( + conn, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp + ) except Refused: mark("recording") # the UPDATE's transaction rolled back raise @@ -628,7 +613,6 @@ async def commit_enforced( return mark("recording") log("the mode's lock is busy; retrying") - await end_mode_transactions(conn, confirm_lock) if not await wait.pause(): if booting: raise Refused( @@ -648,7 +632,13 @@ async def commit_enforced( ) -async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: +async def _mode_try( + conn: Any, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: """One try including rollback or COMMIT, bounded by its lock and statement waits plus the standby allowance; False after a statement timeout rolls back, TimeoutError at the try's deadline.""" @@ -662,7 +652,22 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo await conn.execute( f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" ) + from server.cli import cell_control + + await cell_control.check_lock(conn) await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + state, first = await read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + observed = None if own is None else (own["current_fp"], own["next_fp"]) + if ( + state != "recording" + or first != (True, expected_cell) + or json.dumps(observed, sort_keys=True) + != json.dumps(expected_fp, sort_keys=True) + ): + raise Refused( + [Refusal("postgres acceptance rows", "changed: stop and rerun attach")] + ) mark("unknown") status = await conn.execute( "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" @@ -684,49 +689,6 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo raise -async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: - """End an earlier run's mode transaction still in flight and wait, for up to - `_END_ROUNDS` checks `_END_PAUSE_S` apart, until its lock clears; Refused when its role - is a superuser and attach's is not, or when a backend still holds the lock after the - last check, naming it.""" - rows: list[Any] = [] - for _ in range(_END_ROUNDS): - rows = list(await conn.fetch(_MODE_HOLDERS)) - if not rows: - return - superuser = bool( - await conn.fetchval("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") - ) - for r in rows: - if r["rolsuper"] and not superuser: - raise Refused( - [ - Refusal( - f"postgres backend {r['pid']}", - f"an earlier mode transaction ({r['application_name'] or 'no name'}, " - f"since {r['xact_start']}) runs as a superuser: a superuser must " - "end it (pg_terminate_backend)", - ) - ] - ) - await confirm_lock() - await conn.fetchval("SELECT pg_terminate_backend($1)", r["pid"]) - await _sleep(_END_PAUSE_S) - if not (rows := list(await conn.fetch(_MODE_HOLDERS))): - return - raise Refused( - [ - Refusal( - f"postgres backend {r['pid']}", - f"still holds loom.tenancy_mode's lock after it was ended " - f"({r['application_name'] or 'no name'}, since {r['xact_start']}): rerun attach " - "once it has gone", - ) - for r in rows - ] - ) - - # -- the durable read ------------------------------------------------------------------- _DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried @@ -742,11 +704,13 @@ class Durable: xid: int -async def durable_read() -> Durable: +async def durable_read(conn: Any) -> Durable: """The mode and the `first` row read in a transaction that emits a transactional logical message, whose commit waits for the synchronous standbys: once it returns, the mode read is on them. Refused when the message took no xid.""" - async with asyncio.timeout(_DURABLE_TRY_S), pg.durable_transaction() as conn: + from server.cli import cell_control + + async with asyncio.timeout(_DURABLE_TRY_S), cell_control.transaction(conn): await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py new file mode 100644 index 0000000000..3c03a75e2a --- /dev/null +++ b/server/cli/cell_control.py @@ -0,0 +1,246 @@ +"""Fenced cell control: one directory claim and the exact local lock-owning backend.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any +from uuid import UUID + +from server.cli import cell_attach_pg as apg +from server.cli.cell_ch import Refusal +from server.cli.cell_ch import Refused as Refused +from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S +from server.settings import settings + +# A control transaction has few metadata statements; COMMIT's standby wait is bounded too. +TRANSACTION_S = 60.0 # not a setting: a single control transaction's deadline +OPEN_S = 75.0 # not a setting: connection initialization and direct-session checks +REQUEST_S = CONTROL_MAX_S + 30.0 # not a setting: directory bound plus probes/connect/close +Call = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] + + +def _refused(reason: str) -> Refused: + return Refused([Refusal("cell control", reason)]) + + +def preconditions(row: dict[str, Any], *, claim: bool = False) -> dict[str, Any]: + """Carry the whole observed registration and exact fingerprint pair.""" + if any(name not in row for name in (*CONTROL_FIELDS, "mutation_generation")): + raise _refused("a router replica is not upgraded for cell control fencing") + try: + generation = str(UUID(row["mutation_generation"])) + except (ValueError, TypeError, AttributeError): + raise _refused( + "a router replica returned no readable cell control generation: upgrade it" + ) from None + values = {name: row[name] for name in CONTROL_FIELDS} + if not claim: + values["mutation_generation"] = generation + return {"expected": values} + + +async def check_lock(conn: Any) -> None: + try: + held = await apg.lock_held(conn) + except Exception as e: # noqa: BLE001 — an unreadable lock ends this ownership + raise _refused( + f"lock session lost ({type(e).__name__}): stop and rerun with a fresh claim" + ) from None + if not held: + raise _refused("attach lock lost: stop and rerun with a fresh claim") + + +async def paired(conn: Any, row: dict[str, Any]) -> None: + """The locked authority and configured bucket name the observed registration.""" + recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") + prefix = settings.document_s3_prefix if settings.document_s3_bucket else "" + if ( + not recorded + or recorded != row.get("deployment_id") + or settings.cell != row.get("cell") + or (settings.document_s3_bucket, prefix) != (row.get("bucket"), row.get("bucket_prefix")) + ): + raise _refused( + "this cell configuration and locked Postgres do not match the directory registration" + ) + + +async def claim(conn: Any, call: Call, row: dict[str, Any]) -> dict[str, Any]: + """Claim once; a lost acknowledgement ends the attempt, never restoring an old token.""" + await check_lock(conn) + await paired(conn, row) + answer = await call("/claim", preconditions(row, claim=True)) + await check_lock(conn) + preconditions(answer) + expected = {name: row[name] for name in CONTROL_FIELDS} + if json.dumps(expected, sort_keys=True) != json.dumps( + {name: answer[name] for name in CONTROL_FIELDS}, sort_keys=True + ): + raise _refused( + "claim returned a different registration or fingerprint pair: stop and rerun" + ) + return answer + + +async def mutate( + conn: Any, call: Call, row: dict[str, Any], path: str, body: dict[str, Any] +) -> dict[str, Any]: + """An unknown response stops the caller; a fresh run must claim and recompute.""" + await check_lock(conn) + answer = await call(path, {**body, **preconditions(row)}) + await check_lock(conn) + return answer + + +@asynccontextmanager +async def transaction(conn: Any) -> AsyncIterator[Any]: + """Check, write and durably commit on the lock backend; cancellation ends ownership.""" + settled = False + tx = conn.transaction() + try: + async with asyncio.timeout(TRANSACTION_S): + await tx.start() + try: + await conn.execute("SET LOCAL lock_timeout = '5s'") + await conn.execute("SET LOCAL statement_timeout = '10s'") + await check_lock(conn) + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + yield conn + except Exception: + await tx.rollback() + settled = True + raise + else: + await tx.commit() + settled = True + finally: + if not settled: + # This includes cancellation during COMMIT/rollback. A later session is a new run. + conn.terminate() + + +@asynccontextmanager +async def session(what: str) -> AsyncIterator[Any]: + """A direct, marked, UTF8 session holding ATTACH for its entire control run.""" + from server.core import pg + + cm = pg.dedicated(statement_timeout_ms=60_000) + conn = None + try: + async with asyncio.timeout(OPEN_S): + conn = await cm.__aenter__() + refused = await apg.direct_refusals(conn, what) + refused += await apg.encoding_refusals(conn) + if refused: + raise Refused(refused) + await apg.take_lock(conn, what) + yield conn + finally: + if conn is not None: + # Use dedicated()'s bounded close; this does not prove the server backend ended. + await cm.__aexit__(None, None, None) + + +async def request( + url: Any, + name: str, + transport: Any, + path: str, + body: dict[str, Any] | None = None, + *, + method: str = "POST", +) -> dict[str, Any]: + """Read or mutate one operator row; gateways and lost replies are unknown outcomes.""" + import httpx + + from server.cli import cell as cli + + endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) + try: + async with ( + asyncio.timeout(REQUEST_S), + httpx.AsyncClient( + timeout=cli._OPERATOR_TIMEOUT, + trust_env=url.scheme == "https", + transport=transport, + headers={"Authorization": f"Bearer {settings.router_operator_token}"}, + ) as client, + ): + response = await client.request(method, endpoint, json=body) + except (httpx.HTTPError, TimeoutError) as e: + raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None + answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) + if answer is None: + raise _refused(cli.not_the_router(response)) + if response.status_code == 404 and answer.get("code") != "NO_SUCH_CELL": + raise _refused("a router replica is not upgraded for cell control fencing") + if response.status_code != 200: + raise _refused( + f"router refused {path or 'the read'} ({response.status_code}): {answer['error']}" + ) + return answer + + +async def _acceptance(conn: Any) -> dict[str, Any]: + return { + "first": dict(row) + if (row := await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first")) + else None, + "mode": await conn.fetchval("SELECT mode FROM loom.tenancy_mode"), + "own": dict(row) + if (row := await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self")) + else None, + } + + +async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: + """The runbook's exact state cleanup, fenced against queued controllers/placements.""" + from server.core.schema import pg_migrate + + async with transaction(conn): + before = await _acceptance(conn) + first = before["first"] + if first is not None and (first["cell"], first["is_first"]) != (row["cell"], row["is_first"]): + raise _refused("the local first row names another registration") + already_open = not any(before.values()) and not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom.tenancy)" + ) + if row["is_first"] and before["mode"] != "recording" and not already_open: + raise _refused("first-cell backout requires local recording mode; enforced goes forward") + row = await claim(conn, call, row) + if row.get("state") != "attaching": + raise _refused("a serving cell goes forward; state cleanup refused") + # Closing and reading placed under one directory row lock excludes an ordinary + # placement queued before the claim. A later failure deliberately leaves it closed. + limited = await mutate(conn, call, row, "/limit", {"closed": True}) + if type(limited.get("placed_orgs")) is not int or limited["placed_orgs"] != 0: + raise _refused("cell holds placements: cleanup refused; the cell remains closed") + async with transaction(conn): + await paired(conn, row) + if json.dumps(await _acceptance(conn), sort_keys=True) != json.dumps( + before, sort_keys=True + ): + raise _refused("local acceptance changed: cleanup refused; the cell remains closed") + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + await conn.execute("ALTER TABLE loom.cell_first DISABLE TRIGGER cell_first_immutable") + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + await conn.execute(f"DELETE FROM loom.{table}") + await conn.execute("ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_immutable") + await conn.execute( + "ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_no_truncate" + ) + # Once state is open, a migrating boot cannot recreate the org triggers. The same + # session still holds ATTACH; a retry repeats the durable state cleanup first. + async with transaction(conn): + await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) + triggers = await conn.fetch( + "SELECT t.tgname, c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'loom' " + "AND t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" + ) + for trigger in triggers: + name, table = (str(trigger[k]).replace('"', '""') for k in ("tgname", "relname")) + await conn.execute(f'DROP TRIGGER "{name}" ON loom."{table}"') diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4a820f652b..c4ac332757 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -39,6 +39,7 @@ import httpx from server.cli import cell as cell_cli +from server.cli import cell_control from server.cli._ch import writes from server.core import cell from server.settings import settings @@ -51,9 +52,6 @@ _WAIT_MAX_S = 86400 # not a setting: a flag's bound -# A bound on the lock session's statements: it reads the catalog and holds the lock. -_LOCK_STATEMENT_MS = 60_000 # not a setting: a bound on one CLI session's reads - _sleep = asyncio.sleep # the seam the retry tests replace @@ -109,14 +107,12 @@ def _args_problem(args: argparse.Namespace) -> str: return "" -async def _read_rows() -> _CellRows: - from server.core import pg, pg_profile - - deployment_id = await pg_profile.deployment_id() - first = await pg.fetchrow("SELECT is_first, cell FROM loom.cell_first") - mode = await pg.fetchval("SELECT mode FROM loom.tenancy_mode") - own = await pg.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - tenancy_rows = await pg.fetchval("SELECT count(*) FROM loom.tenancy") +async def _read_rows(conn: Any, *, count_tenancy: bool = True) -> _CellRows: + deployment_id = str(await conn.fetchval("SELECT deployment_id FROM loom.authority") or "") + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + tenancy_rows = await conn.fetchval("SELECT count(*) FROM loom.tenancy") if count_tenancy else 0 return _CellRows( deployment_id=deployment_id, first=None if first is None else (bool(first["is_first"]), str(first["cell"])), @@ -382,15 +378,23 @@ def _own_rows( async def _write_rows( - rows: _CellRows, answer: dict[str, Any], current: dict[str, Any], nxt: dict[str, Any] | None + conn: Any, + rows: _CellRows, + answer: dict[str, Any], + current: dict[str, Any], + nxt: dict[str, Any] | None, ) -> list[str]: """Write the `first` and mode rows that are missing and the own row where it differs from `current` and `nxt`, in one transaction; the rows written.""" - from server.core import pg - is_first = bool(answer.get("is_first")) written = [] - async with pg.transaction() as conn: + async with cell_control.transaction(conn): + # A state CAS does not need to count every tenancy row again. + fresh = await _read_rows(conn, count_tenancy=False) + before = (rows.deployment_id, rows.first, rows.mode, rows.current, rows.next) + after = (fresh.deployment_id, fresh.first, fresh.mode, fresh.current, fresh.next) + if json.dumps(before, sort_keys=True) != json.dumps(after, sort_keys=True): + raise Refused("local acceptance rows changed: stop and rerun with a fresh claim") if rows.first is None: await conn.execute( "INSERT INTO loom.cell_first (is_first, cell) VALUES ($1, $2)", @@ -403,7 +407,9 @@ async def _write_rows( "INSERT INTO loom.tenancy_mode (mode) VALUES ($1)", str(answer["mode"]) ) written.append("mode") - if (rows.current, rows.next) != (current, nxt): + if json.dumps((rows.current, rows.next), sort_keys=True) != json.dumps( + (current, nxt), sort_keys=True + ): await conn.execute( "INSERT INTO loom.cell_self (current_fp, next_fp) VALUES ($1, $2) " "ON CONFLICT (id) DO UPDATE SET current_fp = EXCLUDED.current_fp, " @@ -452,29 +458,18 @@ async def prepare( transport: httpx.AsyncBaseTransport | None = None, ) -> int: """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" - async with attach_lock(): - return await _prepare_locked(ch, args, router, transport) + async with attach_lock() as conn: + return await _prepare_locked(ch, args, router, transport, conn) @contextlib.asynccontextmanager -async def attach_lock() -> AsyncIterator[None]: - """The attach lock, held on a direct session of the cell's Postgres while the block - runs, after that session's checks: prepare and attach write the same probe nonce, and - one of them runs at a time.""" - from server.cli import cell_attach_pg - from server.cli.cell_ch import Refused as Stopped - from server.core import pg - - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - try: - refusals = await cell_attach_pg.direct_refusals(conn, "prepare") - refusals += await cell_attach_pg.encoding_refusals(conn) - if refusals: - raise Stopped(refusals) - await cell_attach_pg.take_lock(conn, "prepare") - except Stopped as e: - raise Refused(str(e)) from None - yield +async def attach_lock() -> AsyncIterator[Any]: + """Hand the exact lock backend to every acceptance-state read and write.""" + try: + async with cell_control.session("prepare") as conn: + yield conn + except cell_control.Refused as e: + raise Refused(str(e)) from None async def _prepare_locked( @@ -482,8 +477,10 @@ async def _prepare_locked( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None, + conn: Any, ) -> int: - rows = await _read_rows() + async with cell_control.transaction(conn): + rows = await _read_rows(conn) if not rows.deployment_id: raise Refused("this cell's Postgres records no deployment id: start the cell once first") if rows.first is not None and rows.first[1] != settings.cell: @@ -518,7 +515,19 @@ async def _prepare_locked( if problem := _answer_problem(rows, answer, beats): raise Refused(problem) current, nxt = _own_rows(rows, answer, every) - written = await _write_rows(rows, answer, current, nxt) + + async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: + return await cell_control.request(router, args.cell, transport, path, body) + + snapshot = await cell_control.request(router, args.cell, transport, "", method="GET") + snapshot = await cell_control.claim(conn, call, snapshot) + # A queued earlier controller may have committed before this claim. Its returned + # row, including the highest mode, supersedes the registration response. + answer = {**answer, **snapshot} + if problem := _answer_problem(rows, snapshot, beats): + raise Refused(problem) + current, nxt = _own_rows(rows, snapshot, every) + written = await _write_rows(conn, rows, answer, current, nxt) print( json.dumps( { diff --git a/server/router/app.py b/server/router/app.py index 4e24a7c922..39a1ac23c8 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -51,6 +51,7 @@ from contextlib import asynccontextmanager from typing import Any, NamedTuple, Protocol from urllib.parse import parse_qs +from uuid import UUID import httpx from starlette.applications import Starlette @@ -542,7 +543,12 @@ async def set_limit(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected known = {"closed", "placements_max"} if not isinstance(body, dict) or not body or set(body) - known: return _refused(400, "a limit is a JSON object of closed and/or placements_max") @@ -556,7 +562,7 @@ async def set_limit(self, request: Request) -> JSONResponse: return _refused(400, "placements_max is a whole number, 0 or more, or null") return await _operator_answer( lambda: directory.set_limit( - request.path_params["cell"], closed=closed, placements_max=quota + request.path_params["cell"], closed=closed, placements_max=quota, expected=expected ) ) @@ -567,7 +573,12 @@ async def rotate_credential(self, request: Request) -> JSONResponse: return refusal if refusal := _malformed_cell_refusal(request): return refusal - body = await _body(request) + body = _parsed(await _bounded(request, ATTACH_BEGIN_MAX_BYTES)) + if not isinstance(body, dict): + return _refused(400, "this call's body is a JSON object") + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected if not isinstance(body, dict) or set(body) - {"step", "sha256"}: return _refused(400, "a rotation is a JSON object of step and sha256") step, sha = body.get("step"), body.get("sha256", "") @@ -578,7 +589,9 @@ async def rotate_credential(self, request: Request) -> JSONResponse: if step == "next" and _is_operator_hash(sha): return _refused(400, _OPERATOR_HASH_REFUSAL) response = await _operator_answer( - lambda: directory.rotate_credential(request.path_params["cell"], step, sha) + lambda: directory.rotate_credential( + request.path_params["cell"], step, sha, expected=expected + ) ) # This replica reads the hashes again on its next call. A 503 is included: its # change may have committed (BACKEND_UNAVAILABLE). @@ -632,7 +645,7 @@ async def register(self, request: Request) -> JSONResponse: ) async def _operator_body( - self, request: Request, limit: int = MAX_BODY_BYTES + self, request: Request, limit: int = ATTACH_BEGIN_MAX_BYTES ) -> dict[str, Any] | JSONResponse: """The operator call's JSON object, or its refusal: the credential, then the cell's name, then the body (413 past `limit`, 400 when it is no JSON object).""" @@ -660,6 +673,19 @@ async def operator_cell(self, request: Request) -> JSONResponse: lambda found: {**_attach_answer(found), "other": found.other}, ) + async def claim(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body, claim=True) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a control claim contains only expected") + return await _answered( + lambda: directory.claim(request.path_params["cell"], expected), _attach_answer + ) + async def attach_begin(self, request: Request) -> JSONResponse: """`attach/begin`: once the probe listener echoes attach's nonces from the stores the row names and the upstream reports the row's deployment id, write when attach @@ -668,6 +694,9 @@ async def attach_begin(self, request: Request) -> JSONResponse: body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected name = request.path_params["cell"] parsed = _begin(body) if isinstance(parsed, JSONResponse): @@ -676,17 +705,29 @@ async def attach_begin(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found - if reason := directory.begin_refusal(found, begin): + if reason := directory.control_refusal(found, expected) or directory.begin_refusal( + found, begin + ): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal - return await _answered(lambda: directory.attach_begin(name, begin), _attach_answer) + return await _answered( + lambda: directory.attach_begin(name, begin, expected=expected), _attach_answer + ) async def attach_place(self, request: Request) -> JSONResponse: """`attach/place {deployment_id, orgs}`: a batch of at most ATTACH_BATCH orgs.""" body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if len(json.dumps(body, ensure_ascii=True).encode()) > MAX_BODY_BYTES: + return _refused( + 413, + f"a placement batch without its control envelope is at most {MAX_BODY_BYTES} bytes", + ) orgs = body.get("orgs") deployment_id = _text(body, "deployment_id", needed=True) if ( @@ -704,7 +745,7 @@ async def attach_place(self, request: Request) -> JSONResponse: name = request.path_params["cell"] batch = [o for o in named if o is not None] return await _answered( - lambda: directory.attach_place(name, deployment_id, batch), + lambda: directory.attach_place(name, deployment_id, batch, expected=expected), lambda counts: {"cell": name, "placed": counts[0], "placed_orgs": counts[1]}, ) @@ -713,12 +754,16 @@ async def attach_enforced(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) if set(body) != {"deployment_id"} or deployment_id is None: return _refused(400, "attach/enforced is a JSON object of deployment_id") name = request.path_params["cell"] return await _answered( - lambda: directory.attach_enforced(name, deployment_id), _attach_answer + lambda: directory.attach_enforced(name, deployment_id, expected=expected), + _attach_answer, ) async def attach_serving(self, request: Request) -> JSONResponse: @@ -728,6 +773,9 @@ async def attach_serving(self, request: Request) -> JSONResponse: body = await self._operator_body(request) if isinstance(body, JSONResponse): return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected deployment_id = _text(body, "deployment_id", needed=True) nonces: Any = body.get("nonces") if set(body) != {"deployment_id", "nonces"} or deployment_id is None: @@ -738,13 +786,15 @@ async def attach_serving(self, request: Request) -> JSONResponse: found = await _read_attaching(name) if isinstance(found, JSONResponse): return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) if not found.served(deployment_id): if reason := directory.attach_refusal(found, deployment_id, enforced=True): return _refused(409, reason) if refusal := await self._stores_check(found, nonces): return refusal response = await _answered( - lambda: directory.attach_serving(name, deployment_id), _attach_answer + lambda: directory.attach_serving(name, deployment_id, expected=expected), _attach_answer ) # A 503's change may have committed: this replica reads the cell's issuers and state # on its next request either way. @@ -760,13 +810,18 @@ async def remove(self, request: Request) -> JSONResponse: a cell `attaching` with no placement (`directory.remove`). The router cannot read the cell's Postgres: `loom cell remove` checks that a first cell's reads `open`, and a direct call skips that check.""" - if refusal := self._operator(request): - return refusal - if refusal := _malformed_cell_refusal(request): - return refusal + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if body: + return _refused(400, "a removal contains only expected") name = request.path_params["cell"] response = await _answered( - lambda: directory.remove(name), lambda row: {"cell": row.cell, "removed": True} + lambda: directory.remove(name, expected=expected), + lambda row: {"cell": row.cell, "removed": True}, ) # A 503 is included: the removal may have committed (BACKEND_UNAVAILABLE). if response.status_code in (200, 503): @@ -1315,6 +1370,41 @@ async def _answered[T]( return JSONResponse(render(result)) +def _control(body: dict[str, Any], *, claim: bool = False) -> directory.Control | JSONResponse: + """Remove and validate the exact observed row; no caller may select a new token.""" + value = body.pop("expected", None) + if not isinstance(value, dict): + return _refused( + 409, + "cell control fencing requires upgraded control clients and every router replica", + code="CONTROL_UPGRADE_REQUIRED", + ) + keys = set(directory.CONTROL_FIELDS) | (set() if claim else {"mutation_generation"}) + if set(value) != keys: + return _refused( + 400, + "expected contains the complete registration, fingerprint pair and claimed generation", + ) + if type(value["is_first"]) is not bool or any( + not isinstance(value[name], str) or "\x00" in value[name] + for name in directory.CONTROL_IDENTITY + if name != "is_first" + ): + return _refused(400, "expected registration has invalid field types") + if any( + value[name] is not None and not isinstance(value[name], dict) + for name in ("current_fp", "next_fp") + ) or _holds_nul(value): + return _refused(400, "expected fingerprint pair contains JSON objects or null") + generation = None + if not claim: + try: + generation = UUID(value["mutation_generation"]) + except (TypeError, ValueError, AttributeError): + return _refused(400, "expected mutation_generation is a UUID") + return directory.Control({name: value[name] for name in directory.CONTROL_FIELDS}, generation) + + def _limits(row: directory.Cell) -> dict[str, Any]: return { "cell": row.cell, @@ -1344,6 +1434,8 @@ def _attach_answer(found: directory.Attaching) -> dict[str, Any]: "attach_started_at": started.isoformat() if started else None, "current_fp": row.current_fp, "next_fp": row.next_fp, + "mutation_generation": str(row.mutation_generation), + **{name: getattr(found, name) for name in directory._REGISTERED}, } @@ -1374,6 +1466,7 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: methods=["POST"], ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), + Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), Route("/cells/v1/operator/cells/{cell}", router.remove, methods=["DELETE"]), Route( "/cells/v1/operator/cells/{cell}/attach/begin", diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..185c5126d6 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -25,6 +25,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Any +from uuid import UUID from server.core import cell as cell_state from server.core.paths import ROUTER_MIGRATIONS_DIR @@ -59,9 +60,9 @@ def _bound_s(statements: int) -> int: return STATEMENT_TIMEOUT_S + (statements + 1) * (STATEMENT_TIMEOUT_S + 1) -# The longest a limit or credential change can take: eight statements (begin, lock, row, -# check, update, audit, read-back, commit). Registration has its own bound, below. -OPERATOR_CHANGE_MAX_S = _bound_s(8) # not a setting: derived from the bound above +# The longest limit/credential change has ten statements: begin, directory lock, row, +# other row, WAL, credential check, update, audit, read-back and commit. +OPERATOR_CHANGE_MAX_S = _bound_s(10) # not a setting: derived from the bound above # The longest a registration's directory write can take: thirteen statements (begin, lock, # row, five refusal reads, three inserts, read-back, commit); and its read before the probe: # six statements (the row, the first cell, then the credential, deployment id, host and @@ -72,11 +73,11 @@ def _bound_s(statements: int) -> int: # The attach calls' statements, which tests/unit/test_router_attach.py counts against the # code: the operator's read of a cell's row (the row, any other row, audit); the read before # `begin`'s and `serving`'s probe (the row, any other row); `begin`, `enforced` and -# `serving` (begin, lock, row, other row, update, audit, read-back, commit); a batch (begin, -# lock, row, other row, insert, placements elsewhere, count, audit, commit); and the removal -# of an unattached cell (begin, lock, row, delete hosts, delete row, audit, commit). +# `serving` (begin, lock, row, other row, WAL, update, audit, read-back, commit); a batch +# (begin, lock, row, other row, WAL, insert, placements elsewhere, count, audit, commit); +# removal adds other-row and WAL reads before deleting hosts/row and auditing. OPERATOR_READ_STATEMENTS, ATTACH_READ_STATEMENTS = 3, 2 -ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 8, 9, 7 +ATTACH_CHANGE_STATEMENTS, ATTACH_PLACE_STATEMENTS, REMOVE_STATEMENTS = 9, 10, 9 OPERATOR_READ_MAX_S = _bound_s(OPERATOR_READ_STATEMENTS) # not a setting: derived REMOVE_MAX_S = _bound_s(REMOVE_STATEMENTS) # not a setting: derived # Every directory connection commits through the synchronous standby and reads what @@ -107,6 +108,7 @@ class Cell: next_fp: dict[str, Any] | None # The new credential's hash during a rotation (`loom cell credential --next`). credential_next_sha256: str | None + mutation_generation: UUID def holds(self, presented: str) -> bool: """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" @@ -231,7 +233,7 @@ async def migrate() -> list[str]: # The columns a Cell holds (read by name: `_cell`). _CELL_COLUMNS = ( "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " - "credential_next_sha256, deployment_id, current_fp, next_fp" + "credential_next_sha256, deployment_id, current_fp, next_fp, mutation_generation" ) _CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" @@ -302,6 +304,7 @@ async def set_limit( *, closed: bool | _Unset = UNSET, placements_max: int | None | _Unset = UNSET, + expected: Control, ) -> Cell: """`loom cell limit`: close or reopen a cell to new orgs and set or clear its placement quota, on its row alone, audited; the cell's stores are not touched. A @@ -324,10 +327,12 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return detail - return await _operator_change(cell_name, "limit", change) + return await _operator_change(cell_name, "limit", change, expected) -async def rotate_credential(cell_name: str, step: str, sha256: str = "") -> Cell: +async def rotate_credential( + cell_name: str, step: str, sha256: str = "", *, expected: Control +) -> Cell: """`loom cell credential`: `next` records a new credential's hash beside the current one, refused when any cell holds it as current or next and while one is pending; `finish` makes `next` current and drops the old; `revert` drops `next`. @@ -365,23 +370,18 @@ async def change(conn: Any, row: Cell) -> dict[str, Any]: ) return {"step": step, **({"sha256": sha256} if sha256 else {})} - return await _operator_change(cell_name, "credential", change) + return await _operator_change(cell_name, "credential", change, expected) async def _operator_change( - cell_name: str, action: str, change: Callable[[Any, Cell], Awaitable[dict[str, Any]]] + cell_name: str, + action: str, + change: Callable[[Any, Cell], Awaitable[dict[str, Any]]], + expected: Control, ) -> Cell: - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if locked is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - detail = await change(conn, _cell(locked)) - await conn.execute( - "INSERT INTO loom_router.audit (actor, action, detail) VALUES ('operator', $1, $2)", - action, - {"cell": cell_name, **detail}, - ) + async with _attach_change(cell_name, expected) as (conn, found): + detail = await change(conn, found.row) + await _audit(conn, action, {"cell": cell_name, **detail}) row = await conn.fetchrow(f"{_CELL} WHERE cell = $1", cell_name) return _cell(row) @@ -866,17 +866,86 @@ async def attach_read(cell_name: str, *, audited: bool = False) -> Attaching: return _attaching(found, other) +# Every acceptance controller compares the full registration and exact JSON pair. +CONTROL_IDENTITY = ("cell", "is_first", "host", "deployment_id", *_REGISTERED) +CONTROL_FIELDS = (*CONTROL_IDENTITY, "current_fp", "next_fp") +CONTROL_MAX_S = _bound_s(10) # not a setting: transaction, commit and pool release bound + + +@dataclass(frozen=True) +class Control: + """The operator's observed registration and pair, with its claimed generation.""" + + values: dict[str, Any] + generation: UUID | None + + +def control_refusal(found: Attaching, expected: Control, *, claim: bool = False) -> str: + if not claim and expected.generation != found.row.mutation_generation: + return "cell control generation changed: stop and rerun the command with a fresh claim" + actual = { + name: getattr(found.row, name) if hasattr(found.row, name) else getattr(found, name) + for name in CONTROL_FIELDS + } + if _as_json(actual) != _as_json(expected.values): + return ( + "cell registration or fingerprints changed: stop and rerun from a fresh directory read" + ) + return "" + + @asynccontextmanager -async def _attach_change(cell_name: str) -> AsyncIterator[tuple[Any, Attaching]]: - """A directory transaction holding the directory lock and then the cell's row lock, with - the row and any other row's name.""" - async with _reading(), _connection() as conn, conn.transaction(): +async def _control_transaction() -> AsyncIterator[Any]: + """Bound acquisition, statements, COMMIT and release; never await rollback on cancel.""" + async with _reading(), asyncio.timeout(CONTROL_MAX_S), _connection() as conn: + settled = False + tx = conn.transaction() + try: + await tx.start() + try: + yield conn + except Exception: + await tx.rollback() + settled = True + raise + else: + await tx.commit() + settled = True + finally: + if not settled: + # A cancelled/unknown COMMIT cannot return this backend to the pool. + conn.terminate() + + +@asynccontextmanager +async def _attach_change( + cell_name: str, expected: Control, *, claim: bool = False +) -> AsyncIterator[tuple[Any, Attaching]]: + async with _control_transaction() as conn: await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) found = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1 FOR UPDATE", cell_name) if found is None: raise NoSuchCell(f"cell {cell_name!r} is not in the directory") other = await conn.fetchval(_OTHER, cell_name) - yield conn, _attaching(found, other) + attaching = _attaching(found, other) + if reason := control_refusal(attaching, expected, claim=claim): + raise OperatorRefused(reason) + # A no-op still has a synchronous WAL commit: a primary-visible earlier write + # whose acknowledgement was lost is not yet a durability proof. + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + yield conn, attaching + + +async def claim(cell_name: str, expected: Control) -> Attaching: + """Rotate on EVERY execution, including an identical HTTP replay.""" + async with _attach_change(cell_name, expected, claim=True) as (conn, _found): + await conn.execute( + "UPDATE loom_router.cells SET mutation_generation = gen_random_uuid() WHERE cell = $1", + cell_name, + ) + await _audit(conn, "control_claim", {"cell": cell_name}) + written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) + return _attaching(written, _found.other) async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: @@ -887,11 +956,11 @@ async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: ) -async def attach_begin(cell_name: str, begin: Begin) -> Attaching: +async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> Attaching: """`attach/begin`: write `attach_started_at` once, the fingerprints while the row has none, and the root-secret fingerprint on every call, audited with the allow lists. Idempotent: a rerun posting the same fingerprints writes only `keys_fp`.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := begin_refusal(found, begin): raise OperatorRefused(reason) await conn.execute( @@ -917,7 +986,9 @@ async def attach_begin(cell_name: str, begin: Begin) -> Attaching: return _attaching(written, None) -async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> tuple[int, int]: +async def attach_place( + cell_name: str, deployment_id: str, orgs: list[str], *, expected: Control +) -> tuple[int, int]: """`attach/place`: place a batch of the first cell's orgs, once its directory mode is `enforced`, past its closed mark and quota (the orgs are the cell's already); the count it placed now and the cell's count. An org already placed in the cell is skipped, so a @@ -927,7 +998,7 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t reads `enforced`, judges that.""" _refuse_long(orgs) batch = sorted(set(orgs)) - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := attach_refusal(found, deployment_id, enforced=True): raise OperatorRefused(reason) new = await conn.fetch( @@ -969,10 +1040,10 @@ async def attach_place(cell_name: str, deployment_id: str, orgs: list[str]) -> t return len(new), int(placed) -async def attach_enforced(cell_name: str, deployment_id: str) -> Attaching: +async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/enforced`: record directory mode `enforced` (registration wrote `recording`), after which the cell places new orgs live; idempotent.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): if reason := attach_refusal(found, deployment_id): raise OperatorRefused(reason) await conn.execute( @@ -1001,10 +1072,10 @@ def issuers(row: Cell) -> list[str]: return sorted(found) -async def attach_serving(cell_name: str, deployment_id: str) -> Attaching: +async def attach_serving(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/serving`: mark the first cell `serving`, with the issuers its fingerprints accept computed from the row as locked. A `serving` row is answered as it is.""" - async with _attach_change(cell_name) as (conn, found): + async with _attach_change(cell_name, expected) as (conn, found): row = found.row if found.served(deployment_id): await _audit(conn, "attach_serving", {"cell": cell_name, "serving_already": True}) @@ -1035,7 +1106,7 @@ def remove_refusal(cell_name: str, state: str, placed: int) -> str: return "" -async def remove(cell_name: str) -> Cell: +async def remove(cell_name: str, *, expected: Control) -> Cell: """`DELETE /cells/v1/operator/cells/` (`loom cell remove`): remove a cell's row and its host rows, audited, while it is `attaching` with no placement, whatever its directory mode. A placement in flight either commits first (and the removal refuses) or @@ -1043,12 +1114,8 @@ async def remove(cell_name: str) -> Cell: first cell's Postgres reads `open` first, and a direct call skips that check, leaving a first cell still prepared answering its new orgs 503 and unable to register again until the full back-out has run.""" - async with _reading(), _connection() as conn, conn.transaction(): - await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) - found = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) - if found is None: - raise NoSuchCell(f"cell {cell_name!r} is not in the directory") - row = _cell(found) + async with _attach_change(cell_name, expected) as (conn, found): + row = found.row if reason := remove_refusal(cell_name, row.state, row.placed): raise OperatorRefused(reason) await conn.execute("DELETE FROM loom_router.hosts WHERE cell = $1", cell_name) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 3450558fe4..573d8fc85a 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -532,8 +532,17 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) + await cell_attach_pg.take_lock(conn, "mode test") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") await cell_attach_pg.commit_enforced( - conn, ok, ok, cell_attach_pg.Wait(20), logged.append, lambda _m: None + conn, + ok, + ok, + cell_attach_pg.Wait(20), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(own["current_fp"], own["next_fp"]), ) await task assert await _mode() == "enforced" diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py new file mode 100644 index 0000000000..a6799d2064 --- /dev/null +++ b/tests/integration/test_cell_control_fence.py @@ -0,0 +1,613 @@ +"""Control requests remain fenced after their cell's lock backend dies.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import httpx +import pytest + +from server.core import cell +from server.router import app, directory +from server.settings import settings + +TOKEN = "lco_fence-test" + + +@pytest.fixture +async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + admin = await asyncpg.connect(dsn) + name = f"loom_fence_{uuid.uuid4().hex[:12]}" + await admin.execute(f'CREATE DATABASE "{name}"') + own = urlunsplit(urlsplit(dsn)._replace(path=f"/{name}")) + monkeypatch.setattr(settings, "router_pg_dsn", own) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + monkeypatch.setattr(directory, "_pool", None) + conn = await asyncpg.connect(own) + try: + await directory.migrate() + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, deployment_id, credential_sha256, state) " + "VALUES ('first', true, 'first.example.test', 'dep-first', $1, 'attaching')", + "a" * 64, + ) + transport = httpx.ASGITransport(app=app.build_app()) + async with httpx.AsyncClient( + transport=transport, + base_url="http://router", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as client: + yield client, conn, own + finally: + await conn.close() + await directory.close() + await admin.execute(f'DROP DATABASE "{name}" WITH (FORCE)') + await admin.close() + + +def expected(row: dict[str, Any], *, generation: bool = True) -> dict[str, Any]: + out = { + k: row[k] + for k in ( + "cell", + "is_first", + "host", + "deployment_id", + "bucket", + "bucket_prefix", + "probe_url", + "upstream_url", + "current_fp", + "next_fp", + ) + } + if generation: + out["mutation_generation"] = row["mutation_generation"] + return {"expected": out} + + +async def read(client: httpx.AsyncClient) -> dict[str, Any]: + response = await client.get("/cells/v1/operator/cells/first") + assert response.status_code == 200, response.text + return response.json() + + +async def claim(client: httpx.AsyncClient, row: dict[str, Any]) -> dict[str, Any]: + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(row, generation=False) + ) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_stale_http_after_backend_death_and_pair_aba(control_stores, monkeypatch) -> None: + import asyncpg + + client, conn, own = control_stores + a = await asyncpg.connect(own) + b = await asyncpg.connect(own) + release = asyncio.Event() + entered = asyncio.Event() + task = None + try: + await a.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + snapshot = await read(client) + old = await claim(client, snapshot) + real = directory.set_limit + + async def delayed(*args, **kwargs): + entered.set() + await release.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "set_limit", delayed) + task = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(old), "closed": True} + ) + ) + await asyncio.wait_for(entered.wait(), 3) + await conn.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + assert await b.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + newer = await claim(client, await read(client)) + # A complete pair ABA cannot reauthorize A: only the current generation can write. + await conn.execute( + "UPDATE loom_router.cells SET current_fp = '{\"v\":1}', next_fp = '{\"v\":1}'" + ) + await conn.execute("UPDATE loom_router.cells SET current_fp = NULL, next_fp = NULL") + assert newer["mutation_generation"] != old["mutation_generation"] + release.set() + response = await task + assert response.status_code == 409, response.text + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + finally: + release.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +async def test_replayed_claim_lost_ack_and_noop_wrong_generation(control_stores) -> None: + client, conn, _ = control_stores + snapshot = await read(client) + lost = await claim(client, snapshot) # its caller lost this acknowledgement + current = await claim(client, snapshot) # duplicate HTTP execution must rotate again + assert lost["mutation_generation"] != current["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(lost), "closed": False} + ) + assert response.status_code == 409, response.text + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(current), "closed": False} + ) + assert response.status_code == 200, response.text + assert ( + str(await conn.fetchval("SELECT mutation_generation FROM loom_router.cells")) + == current["mutation_generation"] + ) + + +@pytest.fixture +async def local_control(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: + import asyncpg + + from server.core import pg + from tests.integration._pg_database import own_database + + async with own_database() as dsn: + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "document_s3_bucket", "") + conn = await asyncpg.connect(dsn, server_settings={pg.TENANCY_MARKER: "on"}) + await directory._init(conn) + await conn.execute( + "INSERT INTO loom.authority (authority, epoch, deployment_id) VALUES ('postgres', 1, 'dep-first')" + ) + try: + yield conn, dsn + finally: + await conn.close() + await pg.close_pool() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("writer", ["prepare", "attach"]) +async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, writer) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_prepare + + admin, dsn = local_control + dead = await asyncpg.connect(dsn) + await directory._init(dead) + await dead.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + rows = cell_prepare._CellRows("dep-first", None, "", None, None, 0) + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("SELECT pg_terminate_backend($1)", dead.get_server_pid()) + successor = await asyncpg.connect(dsn) + try: + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + if writer == "prepare": + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await cell_prepare._write_rows( + dead, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert not await admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + else: + # A dead controller cannot switch to another session to complete its mode write. + await admin.execute( + "INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')" + ) + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ($1)", fp) + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await apg._mode_try(dead, 100, lambda _m: None, "first", (fp, None)) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await successor.close() + await dead.close() + + +@pytest.mark.asyncio +async def test_same_session_prepare_and_mode_commit(local_control) -> None: + from server.cli import cell_attach_pg as apg + from server.cli import cell_control, cell_prepare + + conn, _ = local_control + await apg.take_lock(conn, "test") + fp = cell.as_value({"deployment_id": "dep-first"}) + rows = await cell_prepare._read_rows(conn) + written = await cell_prepare._write_rows( + conn, rows, {"is_first": True, "mode": "recording"}, fp, None + ) + assert written == ["first", "mode", "own"] + assert await apg.lock_held(conn) + assert await apg._mode_try(conn, 100, lambda _m: None, "first", (fp, None)) + result = await apg.durable_read(conn) + assert result.mode == "enforced" and result.first == (True, "first") + assert await apg.lock_held(conn) + # No-op prepare remains a same-backend synchronous durability boundary. + rows = await cell_prepare._read_rows(conn) + assert ( + await cell_prepare._write_rows(conn, rows, {"is_first": True, "mode": "enforced"}, fp, None) + == [] + ) + async with cell_control.transaction(conn): + assert await conn.fetchval("SELECT pg_backend_pid()") == conn.get_server_pid() + + +@pytest.mark.asyncio +async def test_recreated_row_fences_previous_generation(control_stores) -> None: + client, conn, _ = control_stores + previous = await claim(client, await read(client)) + response = await client.request( + "DELETE", "/cells/v1/operator/cells/first", json=expected(previous) + ) + assert response.status_code == 200, response.text + await conn.execute( + "INSERT INTO loom_router.cells(cell, is_first, host, state, deployment_id, credential_sha256) VALUES ('first', true, 'first.example.test', 'attaching', 'dep-first', $1)", + "a" * 64, + ) + recreated = await read(client) + assert recreated["mutation_generation"] != previous["mutation_generation"] + response = await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(previous), "closed": True} + ) + assert response.status_code == 409 + assert not await conn.fetchval("SELECT closed FROM loom_router.cells") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field,value", + [("current_fp", {"v": 1, "fields": {"x": False}}), ("host", "other.example.test")], +) +async def test_claim_checks_exact_pair_and_registration(control_stores, field, value) -> None: + client, conn, _ = control_stores + original = await read(client) + if field == "current_fp": + await conn.execute('UPDATE loom_router.cells SET current_fp = \'{"v":1,"fields":{"x":0}}\'') + original[field] = value + response = await client.post( + "/cells/v1/operator/cells/first/claim", json=expected(original, generation=False) + ) + assert response.status_code == 409 + + +@pytest.mark.asyncio +async def test_missing_generation_requires_upgrade_after_auth(control_stores) -> None: + client, conn, _ = control_stores + for path in ( + "limit", + "credential", + "attach/begin", + "attach/place", + "attach/enforced", + "attach/serving", + ): + response = await client.post(f"/cells/v1/operator/cells/first/{path}", json={}) + assert (response.status_code, response.json()["code"]) == (409, "CONTROL_UPGRADE_REQUIRED") + response = await client.post( + f"/cells/v1/operator/cells/first/{path}", + content=b"broken", + headers={"Authorization": "Bearer lco_wrong"}, + ) + assert response.status_code == 401 + assert await conn.fetchval("SELECT count(*) FROM loom_router.audit") == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("aba", [False, True]) +async def test_inflight_finalizer_cannot_release_after_new_owner_rewidens( + control_stores, + local_control, + aba, +) -> None: + """Use the production mutation guard with a test-only narrowing HTTP action. + + The update command is a later concern. This exercises its directory boundary without + introducing an update route or implementation in this foundation. + """ + import asyncpg + from starlette.responses import JSONResponse + from starlette.routing import Route + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local_admin, local_dsn = local_control + current = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["old"]}} + nxt = {"v": 1, "fields": {"deployment_id": "dep-first", "bearer_issuers": ["new"]}} + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute( + "INSERT INTO loom.cell_self(current_fp, next_fp) VALUES ($1, $2)", nxt, None + ) + a = await asyncpg.connect(local_dsn) + b = await asyncpg.connect(local_dsn) + await directory._init(b) + started, resume = asyncio.Event(), asyncio.Event() + task = None + router = app.Router() + + async def finalizer(request): + body = await router._operator_body(request) + if isinstance(body, JSONResponse): + return body + compare = app._control(body) + if isinstance(compare, JSONResponse): + return compare + started.set() + await resume.wait() # request already sent; caller can no longer retract it + + async def change(): + async with directory._attach_change("first", compare) as (conn, found): + await conn.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL, issuers = ARRAY['new'] WHERE cell = 'first'" + ) + return found + + return await app._answered(change, app._attach_answer) + + client._transport.app.routes.insert( + 0, Route("/test-finalize/{cell}", finalizer, methods=["POST"]) + ) + try: + await apg.take_lock(a, "old controller") + old = await claim(client, await read(client)) + task = asyncio.create_task(client.post("/test-finalize/first", json=expected(old))) + await asyncio.wait_for(started.wait(), 3) + await local_admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "new controller") + newer = await claim(client, await read(client)) + # New owner claims BEFORE widening local acceptance, on the owned backend. + async with cell_control.transaction(b): + await b.execute("UPDATE loom.cell_self SET current_fp = $1, next_fp = $2", current, nxt) + if aba: + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL" + ) + await directory_admin.execute( + "UPDATE loom_router.cells SET current_fp = $1::jsonb, next_fp = $2::jsonb", + json.dumps(current), + json.dumps(nxt), + ) + assert newer["mutation_generation"] != old["mutation_generation"] + resume.set() + response = await task + assert response.status_code == 409, response.text + assert (await read(client))["next_fp"] == nxt + assert await local_admin.fetchval("SELECT next_fp FROM loom.cell_self") == nxt + finally: + resume.set() + if task is not None: + await task + await a.close() + await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("placement_first", [False, True]) +async def test_backout_closure_serializes_queued_placement( + control_stores, + local_control, + placement_first, +) -> None: + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, directory_dsn = control_stores + local_admin, _ = local_control + await directory_admin.execute("UPDATE loom_router.cells SET mode = 'enforced'") + await local_admin.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local_admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local_admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local_admin, "backout test") + original = await read(client) + blocker = await asyncpg.connect(directory_dsn) + placement = cleanup = None + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + if response.status_code != 200: + raise cell_control._refused(response.json()["error"]) + return response.json() + + async def wait_for_lock(key): + for _ in range(100): + count = await directory_admin.fetchval( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND classid::bigint = $1 AND NOT granted", + key, + ) + if count: + return + await asyncio.sleep(0.01) + pytest.fail("owned test request never reached its lock") + + tx = blocker.transaction() + await tx.start() + try: + if placement_first: + await blocker.fetchrow( + "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + # The placement has its org lock and is queued at the cell row before claim. + for _ in range(100): + if await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid::bigint=$1 AND granted)", + directory._PLACE_LOCK, + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement did not acquire its org lock") + cleanup = asyncio.create_task(cell_control.backout(local_admin, call, original)) + for _ in range(100): + waiting = await directory_admin.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " + "AND application_name = 'loom-router' AND wait_event_type = 'Lock'" + ) + if waiting >= 2: + break + await asyncio.sleep(0.01) + else: + pytest.fail("placement and claim did not both queue behind the cell row") + await tx.commit() + assert await placement == ("first", "dep-first") + with pytest.raises(cell_control.Refused, match="holds placements"): + await cleanup + assert await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await local_admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + else: + await blocker.execute( + "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" + ) + placement = asyncio.create_task( + directory.place("queued", "first", app.placement_refusal) + ) + await wait_for_lock(directory._PLACE_LOCK) + await cell_control.backout(local_admin, call, original) + assert not await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + await tx.commit() + with pytest.raises(directory.PlacementRefused, match="closed"): + await placement + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + assert (await read(client))["placed_orgs"] == int(placement_first) + finally: + if blocker.is_in_transaction(): + await tx.rollback() + for task in (placement, cleanup): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception: # noqa: BLE001 — preserve the test failure while reaping owned tasks + pass + await blocker.close() + + +@pytest.mark.asyncio +async def test_registration_rerun_keeps_claim_and_prior_column_lists_still_work(control_stores): + client, conn, _ = control_stores + held = await claim(client, await read(client)) + reg = directory.Registration( + cell="first", + is_first=True, + host="first.example.test", + deployment_id="dep-first", + credential_sha256="a" * 64, + bucket="", + bucket_prefix="", + probe_url="", + tenancy_rows=0, + state_rows=0, + prefix_objects=0, + processes_without_cell=0, + ) + row, written = await directory.register(reg) + assert not written and str(row.mutation_generation) == held["mutation_generation"] + # The prior data plane uses explicit lists and does not interpret the added column. + legacy = await conn.fetchrow( + "SELECT cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, credential_next_sha256, deployment_id, current_fp, next_fp FROM loom_router.cells" + ) + assert legacy["cell"] == "first" and "mutation_generation" not in legacy + await conn.execute( + "INSERT INTO loom_router.cells(cell,host,state,credential_sha256,deployment_id) VALUES ('later','later.example.test','attaching',$1,'dep-later')", + "b" * 64, + ) + generations = await conn.fetch("SELECT mutation_generation FROM loom_router.cells") + assert len({r["mutation_generation"] for r in generations}) == 2 + + +@pytest.mark.asyncio +async def test_attach_lock_dies_between_confirmation_and_mode_write(local_control): + """The ready callback is a real scheduling gap after the lock check.""" + import argparse + + import asyncpg + + from server.cli import cell_attach + from server.cli import cell_attach_pg as apg + + admin, dsn = local_control + fp = cell.as_value({"deployment_id": "dep-first"}) + await admin.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await admin.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await admin.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + a, b = await asyncpg.connect(dsn), await asyncpg.connect(dsn) + await apg.take_lock(a, "old attach") + + class Router: + async def call(self, path, body): + assert path == "/attach/enforced" + return {"cell": "first"} + + run = cell_attach.Run( + None, argparse.Namespace(cell="first", wait=0), Router(), lambda _text: None + ) + run.lock_conn = a + run.mode = "recording" + run.own_fp = (fp, None) + run.row = { + "cell": "first", + "is_first": True, + "host": "first.example.test", + "deployment_id": "dep-first", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": fp, + "next_fp": None, + "mutation_generation": str(uuid.UUID(int=1)), + } + + async def ready(): + await admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + await apg.take_lock(b, "successor") + return [] + + run.ready_for_commit = ready + try: + with pytest.raises( + (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) + ): + await run.commit_mode() + assert await apg.lock_held(b) + assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + finally: + await a.close() + await b.close() diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index caf092e9ec..cbb96ba4cc 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -22,12 +22,31 @@ from server.settings import settings +async def _observed(name: str) -> directory.Control: + try: + found = await directory.attach_read(name) + except directory.NoSuchCell: + return directory.Control({}, uuid.UUID(int=0)) + values = { + key: getattr(found.row, key) if hasattr(found.row, key) else getattr(found, key) + for key in directory.CONTROL_FIELDS + } + return directory.Control(values, found.row.mutation_generation) + + @pytest.fixture -def dsn(monkeypatch: pytest.MonkeyPatch) -> str: - value = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") +async def dsn(monkeypatch: pytest.MonkeyPatch): + from tests.integration._pg_database import create_database, drop_database + + admin_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") + name, value = await create_database(admin_dsn) monkeypatch.setattr(settings, "router_pg_dsn", value) monkeypatch.setattr(directory, "_pool", None) - return value + try: + yield value + finally: + await directory.close() + await drop_database(admin_dsn, name) @pytest.fixture @@ -228,52 +247,72 @@ async def row(cell: str) -> dict[str, Any]: # The operator's changes: a limit, and a credential rotation with its refusals. # Each writes the fields it names and updated_at, and leaves every other column. before = await row(b) - limited = await directory.set_limit(b, closed=True, placements_max=7) + limited = await directory.set_limit( + b, closed=True, placements_max=7, expected=await _observed(b) + ) assert (limited.closed, limited.placements_max) == (True, 7) after = await row(b) assert after["updated_at"] > before["updated_at"] changed = {"closed": True, "placements_max": 7, "updated_at": after["updated_at"]} assert after == {**before, **changed} - cleared = await directory.set_limit(b, placements_max=None) + cleared = await directory.set_limit(b, placements_max=None, expected=await _observed(b)) assert (cleared.closed, cleared.placements_max) == (True, None) # closed is kept assert await row(b) == {**after, "placements_max": None, "updated_at": ANY} with pytest.raises(ValueError): - await directory.set_limit(b) # nothing to set: no statement + await directory.set_limit( + b, expected=await _observed(b) + ) # nothing to set: no statement with pytest.raises(ValueError): - await directory.rotate_credential(b, "finsh") # no such step: nothing written + await directory.rotate_credential( + b, "finsh", expected=await _observed(b) + ) # no such step: nothing written with pytest.raises(ValueError): - await directory.rotate_credential(b, "next") # no hash to record + await directory.rotate_credential( + b, "next", expected=await _observed(b) + ) # no hash to record with pytest.raises(ValueError): - await directory.rotate_credential(b, "finish", "c" * 64) # a hash finish ignores + await directory.rotate_credential( + b, "finish", "c" * 64, expected=await _observed(b) + ) # a hash finish ignores with pytest.raises(directory.NoSuchCell): - await directory.set_limit(f"nope-{b}", closed=True) + await directory.set_limit( + f"nope-{b}", closed=True, expected=await _observed(f"nope-{b}") + ) sha_b = hashlib.sha256(b.encode()).hexdigest() with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(a, "next", sha_b) # b's current hash + await directory.rotate_credential( + a, "next", sha_b, expected=await _observed(a) + ) # b's current hash with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(a, "finish") + await directory.rotate_credential(a, "finish", expected=await _observed(a)) new = hashlib.sha256(f"new-{a}".encode()).hexdigest() before = await row(a) - assert (await directory.rotate_credential(a, "next", new)).credential_next_sha256 == new + assert ( + await directory.rotate_credential(a, "next", new, expected=await _observed(a)) + ).credential_next_sha256 == new assert await row(a) == {**before, "credential_next_sha256": new, "updated_at": ANY} assert (await directory.credentials())[a] == (hashlib.sha256(a.encode()).hexdigest(), new) with pytest.raises(directory.OperatorRefused, match="already cell .* next credential"): - await directory.rotate_credential(a, "next", new) # this change, retried + await directory.rotate_credential( + a, "next", new, expected=await _observed(a) + ) # this change, retried with pytest.raises(directory.OperatorRefused, match="another next credential pending"): - await directory.rotate_credential(a, "next", "e" * 64) + await directory.rotate_credential(a, "next", "e" * 64, expected=await _observed(a)) with pytest.raises(directory.OperatorRefused, match="records that credential hash"): - await directory.rotate_credential(b, "next", new) # a's next hash + await directory.rotate_credential( + b, "next", new, expected=await _observed(b) + ) # a's next hash # Two cells asking for one next hash at once: the directory lock lets one in. raced = hashlib.sha256(f"raced-{a}".encode()).hexdigest() both = await asyncio.gather( - directory.rotate_credential(b, "next", raced), - directory.rotate_credential(q, "next", raced), + directory.rotate_credential(b, "next", raced, expected=await _observed(b)), + directory.rotate_credential(q, "next", raced, expected=await _observed(q)), return_exceptions=True, ) assert sum(isinstance(r, directory.Cell) for r in both) == 1 assert sum(isinstance(r, directory.OperatorRefused) for r in both) == 1 pending = await row(a) - finished = await directory.rotate_credential(a, "finish") + finished = await directory.rotate_credential(a, "finish", expected=await _observed(a)) assert (finished.credential_sha256, finished.credential_next_sha256) == (new, None) assert await row(a) == { **pending, @@ -282,11 +321,13 @@ async def row(cell: str) -> dict[str, Any]: "updated_at": ANY, } winner = b if isinstance(both[0], directory.Cell) else q - reverted = await directory.rotate_credential(winner, "revert") + reverted = await directory.rotate_credential( + winner, "revert", expected=await _observed(winner) + ) assert reverted.credential_next_sha256 is None assert (await directory.credentials())[winner] == (reverted.credential_sha256,) with pytest.raises(directory.OperatorRefused, match="no next credential"): - await directory.rotate_credential(winner, "revert") + await directory.rotate_credential(winner, "revert", expected=await _observed(winner)) def steps(cell: str) -> Any: return conn.fetch( @@ -587,16 +628,22 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) try: # Nothing before begin; begin writes its start once and the fingerprints. with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="has not begun"): - await directory.attach_place(name, dep, ["a"]) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) with pytest.raises(directory.OperatorRefused, match="deployment id"): - await directory.attach_begin(name, directory.Begin("dep-x", _fp(name), None, "k", {})) - found = await directory.attach_begin(name, begin) + await directory.attach_begin( + name, + directory.Begin("dep-x", _fp(name), None, "k", {}), + expected=await _observed(name), + ) + found = await directory.attach_begin(name, begin, expected=await _observed(name)) started = found.attach_started_at assert started is not None and found.row.current_fp == _fp(name) again = await directory.attach_begin( - name, directory.Begin(dep, _fp(name), None, "keys-2", {}) + name, + directory.Begin(dep, _fp(name), None, "keys-2", {}), + expected=await _observed(name), ) assert again.attach_started_at == started stored = await conn.fetchrow( @@ -609,35 +656,45 @@ async def test_the_first_cell_attaches_through_the_operator_calls(own_dsn: str) ) with pytest.raises(directory.OperatorRefused, match="loom cell update"): await directory.attach_begin( - name, directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}) + name, + directory.Begin(dep, _fp(name, oidc_issuer="x"), None, "k", {}), + expected=await _observed(name), ) # A batch waits for the directory mode `enforced`. with pytest.raises(directory.OperatorRefused, match="attach/enforced first"): - await directory.attach_place(name, dep, ["a"]) - enforced = await directory.attach_enforced(name, dep) + await directory.attach_place(name, dep, ["a"], expected=await _observed(name)) + enforced = await directory.attach_enforced(name, dep, expected=await _observed(name)) assert enforced.row.mode == "enforced" - await directory.attach_enforced(name, dep) # idempotent + await directory.attach_enforced(name, dep, expected=await _observed(name)) # idempotent # A closed cell with a quota of 0 still takes its own orgs; a rerun places none # twice. await conn.execute( "UPDATE loom_router.cells SET closed = true, placements_max = 0 WHERE cell = $1", name ) - assert await directory.attach_place(name, dep, ["b", "a", "a"]) == (2, 2) - assert await directory.attach_place(name, dep, ["a", "c"]) == (1, 3) + assert await directory.attach_place( + name, dep, ["b", "a", "a"], expected=await _observed(name) + ) == (2, 2) + assert await directory.attach_place( + name, dep, ["a", "c"], expected=await _observed(name) + ) == (1, 3) placed = await conn.fetch( "SELECT org FROM loom_router.placements WHERE cell = $1 ORDER BY org", name ) assert [r["org"] for r in placed] == ["a", "b", "c"] with pytest.raises(directory.OrgTooLong): - await directory.attach_place(name, dep, ["d", "x" * 2049]) - served = await directory.attach_serving(name, dep) + await directory.attach_place( + name, dep, ["d", "x" * 2049], expected=await _observed(name) + ) + served = await directory.attach_serving(name, dep, expected=await _observed(name)) assert served.row.state == "serving" assert await conn.fetchval( "SELECT issuers FROM loom_router.cells WHERE cell = $1", name ) == [f"https://{name}.example.test"] - assert (await directory.attach_serving(name, dep)).row.state == "serving" # done + assert ( + await directory.attach_serving(name, dep, expected=await _observed(name)) + ).row.state == "serving" # done with pytest.raises(directory.OperatorRefused, match="serving already"): - await directory.attach_enforced(name, dep) + await directory.attach_enforced(name, dep, expected=await _observed(name)) read = await directory.attach_read(name, audited=True) assert (read.row.state, read.other) == ("serving", None) audit = await conn.fetch( @@ -686,8 +743,10 @@ async def _attaching_first(own_dsn: str) -> tuple[str, str]: name = f"lr-{uuid.uuid4().hex[:8]}" dep = f"dep-{name}" await directory.register(_first(name)) - await directory.attach_begin(name, directory.Begin(dep, _fp(name), None, "k", {})) - await directory.attach_enforced(name, dep) + await directory.attach_begin( + name, directory.Begin(dep, _fp(name), None, "k", {}), expected=await _observed(name) + ) + await directory.attach_enforced(name, dep, expected=await _observed(name)) return name, dep @@ -740,7 +799,9 @@ async def _held(*a: Any) -> None: async with asyncio.timeout(30): live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await inserted.wait() - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme", "beta"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme", "beta"], expected=await _observed(name)) + ) await _waits_on_the_row_lock(conn, "FOR UPDATE") release.set() assert await live == (name, dep) @@ -768,8 +829,8 @@ async def test_a_live_placement_racing_a_batch_of_the_same_org_answers_it_placed real = directory._attach_change @contextlib.asynccontextmanager - async def _held(cell_name: str) -> Any: - async with real(cell_name) as locked: + async def _held(cell_name: str, expected: directory.Control) -> Any: + async with real(cell_name, expected) as locked: yield locked written.set() await release.wait() # the batch's writes are done, its commit held @@ -780,7 +841,9 @@ async def _held(cell_name: str) -> Any: # A full quota: a live placement judged on it would be refused. await conn.execute("UPDATE loom_router.cells SET placements_max = 0 WHERE cell = $1", name) async with asyncio.timeout(30): - batch = asyncio.ensure_future(directory.attach_place(name, dep, ["acme"])) + batch = asyncio.ensure_future( + directory.attach_place(name, dep, ["acme"], expected=await _observed(name)) + ) await written.wait() live = asyncio.ensure_future(directory.place("acme", name, lambda _row: "")) await _waits_on_the_row_lock(conn, "FOR UPDATE") @@ -821,23 +884,29 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( later, ) with pytest.raises(directory.OperatorRefused, match=f"loom cell remove --cell {later}"): - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - assert (await directory.remove(later)).cell == later + await directory.attach_begin( + first, + directory.Begin(dep, _fp(first), None, "k", {}), + expected=await _observed(first), + ) + assert (await directory.remove(later, expected=await _observed(later))).cell == later assert ( await conn.fetchval("SELECT count(*) FROM loom_router.hosts WHERE cell = $1", later) == 0 ) with pytest.raises(directory.NoSuchCell): - await directory.remove(later) + await directory.remove(later, expected=await _observed(later)) # The first cell backs out at any directory mode until its first placement. - await directory.attach_begin(first, directory.Begin(dep, _fp(first), None, "k", {})) - await directory.attach_enforced(first, dep) - await directory.attach_place(first, dep, ["only"]) + await directory.attach_begin( + first, directory.Begin(dep, _fp(first), None, "k", {}), expected=await _observed(first) + ) + await directory.attach_enforced(first, dep, expected=await _observed(first)) + await directory.attach_place(first, dep, ["only"], expected=await _observed(first)) with pytest.raises(directory.OperatorRefused, match="never back"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) await conn.execute("UPDATE loom_router.cells SET state = 'serving' WHERE cell = $1", first) with pytest.raises(directory.OperatorRefused, match="stays attached"): - await directory.remove(first) + await directory.remove(first, expected=await _observed(first)) removed = await conn.fetch( "SELECT detail FROM loom_router.audit WHERE action = 'remove' ORDER BY id" ) @@ -860,10 +929,14 @@ async def test_an_attaching_cell_with_no_placement_is_removed_with_its_hosts( await conn.execute("DELETE FROM loom_router.cells") await directory.register(_first(other)) await directory.attach_begin( - other, directory.Begin(f"dep-{other}", _fp(other), None, "k", {}) + other, + directory.Begin(f"dep-{other}", _fp(other), None, "k", {}), + expected=await _observed(other), ) - await directory.attach_enforced(other, f"dep-{other}") # a step-3 lock that never landed - assert (await directory.remove(other)).is_first + await directory.attach_enforced( + other, f"dep-{other}", expected=await _observed(other) + ) # a step-3 lock that never landed + assert (await directory.remove(other, expected=await _observed(other))).is_first assert await conn.fetchval("SELECT count(*) FROM loom_router.cells") == 0 assert await conn.fetchval("SELECT count(*) FROM loom_router.hosts") == 0 finally: diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0ecd3a96b3..d573d37423 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -251,7 +251,16 @@ async def ready() -> list[str]: wait = apg.Wait(4, clock) clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): - await apg.commit_enforced(conn, nothing, ready, wait, lambda _t: None, lambda _m: None) + await apg.commit_enforced( + conn, + nothing, + ready, + wait, + lambda _t: None, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) assert clock.now - 100 <= 4 and wait.spent <= 4 # A 2 s try, a 1 s pause, then a try that waits only the second left. assert [a.split("'")[1] for a in asked] == ["2000ms", "1000ms"] @@ -449,6 +458,22 @@ def _answer(self, sql: str, args: tuple[Any, ...]) -> Any: if isinstance(answer, BaseException): raise answer return answer(*args) if callable(answer) else answer + if sql.startswith("SELECT (SELECT is_first FROM loom.cell_first)"): + return { + "is_first": True, + "cell": "first", + "mode": "recording", + "has_own": True, + "has_tenancy": True, + } + if sql == apg._HELD: + return True + if sql == "SELECT mode FROM loom.tenancy_mode": + return "recording" + if sql == "SELECT is_first, cell FROM loom.cell_first": + return {"is_first": True, "cell": "first"} + if sql == "SELECT current_fp, next_fp FROM loom.cell_self": + return {"current_fp": CURRENT, "next_fp": NEXT} raise AssertionError(f"unrouted: {sql[:80]}") async def fetch(self, sql: str, *args: Any) -> Any: @@ -642,7 +667,9 @@ class _Timeout(Exception): @pytest.mark.asyncio -async def test_the_mode_commit_retries_a_busy_lock_and_ends_an_earlier_run(clock: _Clock) -> None: +async def test_the_mode_commit_retries_a_busy_lock_without_ending_other_backends( + clock: _Clock, +) -> None: tries = _seq(_Timeout(), "UPDATE 1") def lock(*_a: Any) -> str: @@ -673,34 +700,22 @@ async def ready() -> list[str]: logged: list[str] = [] await apg.commit_enforced( - conn, confirm, ready, apg.Wait(10, clock), logged.append, lambda _m: None + conn, + confirm, + ready, + apg.Wait(10, clock), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert logged == ["the mode's lock is busy; retrying"] - # Before each try and the one termination. - assert len(confirmed) == 3 + # Before each try, with no termination of a different backend. + assert len(confirmed) == 2 assert conn.executed.count("ROLLBACK") == 1 assert "TERMINATE" not in conn.executed -@pytest.mark.asyncio -async def test_a_superusers_mode_transaction_is_named_not_ended() -> None: - conn = FakeConn( - [ - ( - "l.mode = 'ExclusiveLock'", - [{"pid": 5, "application_name": "psql", "xact_start": "t", "rolsuper": True}], - ), - ("rolsuper FROM pg_roles", False), - ] - ) - - async def confirm() -> None: - pass - - with pytest.raises(Refused, match="a superuser must end it"): - await apg.end_mode_transactions(conn, confirm) - - @pytest.mark.asyncio @pytest.mark.parametrize("wait_s", [0, 2]) async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( @@ -728,6 +743,8 @@ async def confirm() -> None: apg.Wait(wait_s, clock), lambda _t: None, lambda _m: None, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert [str(r) for r in error.value.refusals] == [ "process a: still booting at the end of --wait", @@ -768,7 +785,14 @@ async def ready() -> list[str]: with pytest.raises(TimeoutError): await asyncio.wait_for( apg.commit_enforced( - conn, confirm, ready, apg.Wait(wait_s, clock), lambda _t: None, marks.append + conn, + confirm, + ready, + apg.Wait(wait_s, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ), timeout=0.2, ) @@ -796,7 +820,9 @@ async def execute(self, sql: str, *args: Any) -> str: timeout = asyncio.timeout monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) marks: list[str] = [] - task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) + task = asyncio.create_task( + apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) + ) done, _ = await asyncio.wait([task], timeout=0.2) if not done: task.cancel() @@ -863,7 +889,14 @@ async def ready() -> list[str]: with pytest.raises(KeyboardInterrupt): await apg.commit_enforced( - conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append + conn, + confirm, + ready, + apg.Wait(2, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), ) assert conn.executed[-1] == "TERMINATE" assert marks == [] @@ -885,13 +918,13 @@ async def synchronous() -> str: async def read(mode: str, xid: int) -> apg.Durable: return apg.Durable(mode=mode, first=(True, "first"), xid=xid) - monkeypatch.setattr(apg, "durable_read", lambda: read("recording", 50)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("recording", 50)) with pytest.raises(Refused, match="did not land"): await run.durable() - monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 100)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 100)) with pytest.raises(Refused, match="is not above 100"): await run.durable() - monkeypatch.setattr(apg, "durable_read", lambda: read("enforced", 99)) + monkeypatch.setattr(apg, "durable_read", lambda _conn: read("enforced", 99)) assert await run.durable() == 100 async def waiting() -> str: @@ -962,7 +995,7 @@ async def test_a_stop_between_a_landed_commit_and_the_durable_read_says_the_outc async def dedicated(**_kw: Any) -> Any: yield None - async def landed(*a: Any) -> None: + async def landed(*a: Any, **_kw: Any) -> None: a[-1]("unknown") # the try sent its UPDATE; the commit landed, its answer may not have async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: @@ -976,6 +1009,7 @@ async def run(self: Any) -> int: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(apg, "commit_enforced", landed) monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) monkeypatch.setattr(cell_attach.Run, "run", run) url = httpx.URL("http://127.0.0.1:7900") assert await cell_attach.attach(None, _args(), url) == 1 @@ -1169,7 +1203,16 @@ async def ok() -> list[str]: async def nothing() -> None: pass - await apg.commit_enforced(conn, nothing, ok, apg.Wait(0, clock), lambda _t: None, marks.append) + await apg.commit_enforced( + conn, + nothing, + ok, + apg.Wait(0, clock), + lambda _t: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) assert marks == ["unknown"] assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) @@ -1227,6 +1270,7 @@ async def confirm(_self: Any) -> None: async def run(self: Any) -> int: self.row, self.mode = {"deployment_id": "dep-1"}, "recording" + self.lock_conn = conn self.wait = apg.Wait(2, clock) await self.commit_mode() return 0 @@ -1241,6 +1285,7 @@ async def dedicated(**_kw: Any) -> Any: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(cell_attach.Router, "call", call) + monkeypatch.setattr(cell_attach.Run, "mutate", call) monkeypatch.setattr(cell_attach.Run, "ready_for_commit", ready) monkeypatch.setattr(cell_attach.Run, "confirm_lock", confirm) monkeypatch.setattr(cell_attach.Run, "run", run) @@ -1248,27 +1293,6 @@ async def dedicated(**_kw: Any) -> Any: assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" -@pytest.mark.asyncio -async def test_a_mode_transaction_that_will_not_end_is_named(clock: _Clock) -> None: - held = [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}] - conn = FakeConn( - [ - ("l.mode = 'ExclusiveLock'", held), - ("rolsuper FROM pg_roles", False), - ("pg_terminate_backend", True), - ] - ) - - async def confirm() -> None: - pass - - with pytest.raises(Refused) as e: - await apg.end_mode_transactions(conn, confirm) - (refusal,) = e.value.refusals - assert refusal.subject == "postgres backend 5" - assert "still holds loom.tenancy_mode's lock" in refusal.reason - - def _stub_attach(monkeypatch: pytest.MonkeyPatch, body: Any) -> None: from contextlib import asynccontextmanager @@ -1295,6 +1319,7 @@ async def run(self: Any) -> int: return 0 monkeypatch.setattr(cell_attach.Router, "call", refuse) + monkeypatch.setattr(cell_attach.Run, "mutate", refuse) _stub_attach(monkeypatch, run) assert await cell_attach.attach(None, _args(), httpx.URL("http://127.0.0.1:7900")) == 1 assert capsys.readouterr().err.splitlines()[-1] == "loom cell attach: the cell stays recording" @@ -1309,7 +1334,7 @@ async def test_a_durable_read_that_sees_recording_says_recording( async def synchronous() -> str: return "synchronous" - async def read() -> apg.Durable: + async def read(_conn: Any) -> apg.Durable: return apg.Durable(mode="recording", first=(True, "first"), xid=1) async def run(self: Any) -> int: diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py new file mode 100644 index 0000000000..71d687baf9 --- /dev/null +++ b/tests/unit/test_cell_control.py @@ -0,0 +1,123 @@ +"""A control owner cannot survive an uncertain commit or a lost lock session.""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_control +from tests.unit.test_cell_attach import FakeConn + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", ["COMMIT", "ROLLBACK"]) +async def test_uncertain_transaction_ends_owned_session_without_unbounded_cleanup( + monkeypatch, stall +): + class Stalled(FakeConn): + async def execute(self, sql: str, *args: Any) -> str: + answer = await super().execute(sql, *args) + if sql == stall: + await asyncio.Future() + return answer + + conn = Stalled() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.01) + with pytest.raises(TimeoutError): + async with cell_control.transaction(conn): + if stall == "ROLLBACK": + raise ValueError("refuse") + assert conn.executed[-1] == "TERMINATE" + assert stall in conn.executed + assert any("pg_logical_emit_message" in sql for sql in conn.executed) + + +@pytest.mark.asyncio +async def test_cancellation_during_body_never_waits_for_rollback(): + conn = FakeConn() + entered = asyncio.Event() + + async def work(): + async with cell_control.transaction(conn): + entered.set() + await asyncio.Future() + + task = asyncio.create_task(work()) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert conn.executed[-1] == "TERMINATE" + assert "ROLLBACK" not in conn.executed + + +def snapshot(): + return { + "cell": "first", + "is_first": True, + "host": "first.example", + "deployment_id": "dep", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("after_claim", [False, True]) +async def test_lost_lock_never_reconnects_or_continues_claim(monkeypatch, after_claim): + lock = AsyncMock( + side_effect=[None, cell_control._refused("lost")] + if after_claim + else cell_control._refused("lost") + ) + monkeypatch.setattr(cell_control, "check_lock", lock) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(return_value={**snapshot(), "mutation_generation": str(UUID(int=2))}) + with pytest.raises(cell_control.Refused, match="lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == int(after_claim) + + +@pytest.mark.asyncio +async def test_unknown_claim_acknowledgement_never_reuses_the_observed_generation(monkeypatch): + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_control, "paired", AsyncMock()) + call = AsyncMock(side_effect=OSError("reply lost")) + with pytest.raises(OSError, match="reply lost"): + await cell_control.claim(object(), call, snapshot()) + assert call.await_count == 1 + assert "mutation_generation" not in call.call_args.args[1]["expected"] + + +def test_older_router_snapshot_cannot_make_a_control_request(): + row = snapshot() + row.pop("mutation_generation") + with pytest.raises(cell_control.Refused, match="not upgraded"): + cell_control.preconditions(row) + + +@pytest.mark.asyncio +async def test_whole_router_request_is_bounded_even_without_a_transport_reply(monkeypatch): + import httpx + + async def stalled(_request): + await asyncio.Future() + + monkeypatch.setattr(cell_control, "REQUEST_S", 0.01) + with pytest.raises(cell_control.Refused, match="reply unknown.*TimeoutError"): + await cell_control.request( + httpx.URL("http://127.0.0.1:1"), + "first", + httpx.MockTransport(stalled), + "/claim", + {}, + ) diff --git a/tests/unit/test_cell_operator.py b/tests/unit/test_cell_operator.py index 63b1e00d26..5c37b78cb0 100644 --- a/tests/unit/test_cell_operator.py +++ b/tests/unit/test_cell_operator.py @@ -8,6 +8,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -23,6 +24,29 @@ SHA = "a" * 64 +OBSERVED = { + "cell": "eu-1", + "is_first": False, + "host": "eu-1.example", + "deployment_id": "dep-eu-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), +} + + +@pytest.fixture(autouse=True) +def control_snapshot(monkeypatch: pytest.MonkeyPatch) -> None: + def read(url: str, **_kw: Any) -> httpx.Response: + return httpx.Response(200, json=OBSERVED, request=httpx.Request("GET", url)) + + monkeypatch.setattr(httpx, "get", read) + + def test_a_new_credential_is_lcr_and_128_hex_with_its_sha256(capsys) -> None: credential, sha = cell_cli.new_credential() assert credential.startswith("lcr_") and len(credential) == 132 @@ -110,7 +134,9 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_credential(_args(cell="eu-1", **kw)) == 0 (call,) = operator.calls assert call["url"] == "https://router.test/cells/v1/operator/cells/eu-1/credential" - assert call["json"] == body and call["headers"] == {"Authorization": f"Bearer {OPERATOR}"} + assert call["json"] == {**body, "expected": OBSERVED} and call["headers"] == { + "Authorization": f"Bearer {OPERATOR}" + } @pytest.mark.parametrize( @@ -124,7 +150,7 @@ def test_a_rotation_step_calls_the_operator_api(operator, kw, body) -> None: ) def test_a_limit_calls_the_operator_api(operator, kw, body) -> None: assert cell_cli._cmd_limit(_args(cell="eu-1", **kw)) == 0 - assert operator.calls[0]["json"] == body + assert operator.calls[0]["json"] == {**body, "expected": OBSERVED} assert operator.calls[0]["url"].endswith("/cells/v1/operator/cells/eu-1/limit") @@ -345,7 +371,7 @@ def test_a_page_that_is_not_the_routers_answer_fails(monkeypatch, capsys, comman assert cell_cli._cmd_limit(_args(cell="eu-1", closed=True)) == 1 err = capsys.readouterr().err assert "did not answer as the router does" in err - assert ("the change may have been made" in err) is (command == "limit") + assert "the change may have been made" not in err # failed precondition read sent no mutation def _parse(*argv: str) -> argparse.Namespace: @@ -388,6 +414,7 @@ def _row(**over: Any) -> directory.Cell: "placed": 3, "credential_sha256": "b" * 64, "credential_next_sha256": None, + "mutation_generation": UUID(int=1), "deployment_id": "dep-eu-1", "current_fp": None, "next_fp": None, @@ -403,12 +430,18 @@ def api(monkeypatch) -> tuple[TestClient, list[Any]]: calls: list[Any] = [] async def _limit(cell: str, **kw: Any) -> directory.Cell: + expected = kw.pop("expected") + assert expected.generation == UUID(int=1) + assert expected.values == {k: OBSERVED[k] for k in directory.CONTROL_FIELDS} calls.append(("limit", cell, kw)) if cell == "nope": raise directory.NoSuchCell("cell 'nope' is not in the directory") return _row(closed=kw.get("closed") is True) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate( + cell: str, step: str, sha: str = "", *, expected: directory.Control + ) -> directory.Cell: + assert expected.generation == UUID(int=1) calls.append(("credential", cell, step, sha)) if sha == "c" * 64: raise directory.OperatorRefused("cell 'eu-2' records that credential hash already") @@ -420,6 +453,8 @@ async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: def _post(client: TestClient, path: str, body: Any, token: str = OPERATOR) -> Any: + if isinstance(body, dict): + body = {**body, "expected": OBSERVED} return client.post( f"/cells/v1/operator/cells/{path}", json=body, headers={"Authorization": f"Bearer {token}"} ) @@ -536,7 +571,7 @@ async def _credentials() -> dict[str, tuple[str, ...]]: reads.append(1) return dict(hashes) - async def _rotate(cell: str, step: str, sha: str = "") -> directory.Cell: + async def _rotate(cell: str, step: str, sha: str = "", **_kw: Any) -> directory.Cell: hashes["eu-1"] = ("b" * 64, sha) return _row(credential_next_sha256=sha) diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index c978eb94e2..3071dddccf 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -15,7 +15,7 @@ import pytest from server.cli import cell as cell_cli -from server.cli import cell_prepare +from server.cli import cell_control, cell_prepare from server.core import cell from server.settings import settings @@ -369,7 +369,7 @@ async def test_the_processes_live_after_the_wait_are_the_ones_judged(monkeypatch reads = [[_beat("i-old", before)], [_beat("i-new", after)]] written: list[Any] = [] - async def _rows_read() -> cell_prepare._CellRows: + async def _rows_read(_conn: Any) -> cell_prepare._CellRows: return _rows() async def _beats(_ch: Any) -> list[cell_prepare._Beat]: @@ -381,7 +381,7 @@ async def _nothing(*_a: Any) -> Any: async def _register(*_a: Any) -> dict[str, Any]: return {**OK, "registered": True} - async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: + async def _write(_conn: Any, _rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: written.append((current, nxt)) return ["first", "mode", "own"] @@ -395,6 +395,18 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: ("_write_rows", _write), ): monkeypatch.setattr(cell_prepare, name, fake) + monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + + directory_mode = ["recording"] + + async def snapshot(*_a: Any, **_kw: Any) -> dict[str, Any]: + return {**OK, "mode": directory_mode[0], "current_fp": None, "next_fp": None} + + async def claimed(_conn: Any, _call: Any, row: Any) -> Any: + return row + + monkeypatch.setattr(cell_control, "request", snapshot) + monkeypatch.setattr(cell_control, "claim", claimed) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert written == [(cell.as_value(after), None)] and reads == [] # stdout stays one JSON document; a first cell's next step goes to stderr. @@ -402,6 +414,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert json.loads(out.out)["is_first"] is True assert out.err.strip() == "loom cell prepare: prepared the first cell; roll out LOOM_CELL next" # A rerun on a cell past recording names no next step. + directory_mode[0] = "enforced" monkeypatch.setattr( cell_prepare, "_register", AsyncMock(return_value={**OK, "mode": "enforced"}) ) @@ -409,6 +422,7 @@ async def _write(_rows: Any, _answer: Any, current: Any, nxt: Any) -> list[str]: assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" # A rerun in recording that writes no first row (the rollout is usually done). + directory_mode[0] = "recording" monkeypatch.setattr(cell_prepare, "_register", AsyncMock(return_value={**OK})) async def _own_only(*_a: Any) -> list[str]: @@ -418,3 +432,16 @@ async def _own_only(*_a: Any) -> list[str]: reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 assert capsys.readouterr().err == "" + + # A prior controller committed enforced after registration but before this claim. + directory_mode[0] = "enforced" + posted: list[str] = [] + + async def write_latest(_conn: Any, _rows: Any, answer: Any, *_fp: Any) -> list[str]: + posted.append(answer["mode"]) + return ["mode"] + + monkeypatch.setattr(cell_prepare, "_write_rows", write_latest) + reads.extend([[_beat("i-new", after)], [_beat("i-new", after)]]) + assert await cell_prepare.prepare(None, _args(), ROUTER) == 0 + assert posted == ["enforced"] diff --git a/tests/unit/test_cell_register.py b/tests/unit/test_cell_register.py index 3d26bfa003..16a66b3489 100644 --- a/tests/unit/test_cell_register.py +++ b/tests/unit/test_cell_register.py @@ -9,6 +9,7 @@ import hashlib import json from typing import Any +from uuid import UUID import httpx import pytest @@ -90,6 +91,7 @@ def _row(reg: directory.Registration) -> directory.Cell: current_fp=None, next_fp=None, credential_next_sha256=None, + mutation_generation=UUID(int=1), ) @@ -501,7 +503,23 @@ def test_a_cell_never_registers_with_the_operator_credentials_hash(router) -> No assert r.status_code == 400 and "operator credential's hash" in r.json()["error"] rotated = client.post( "/cells/v1/operator/cells/eu-1/credential", - json={"step": "next", "sha256": operator_sha}, + json={ + "step": "next", + "sha256": operator_sha, + "expected": { + "cell": "eu-1", + "is_first": True, + "host": "loom.example", + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) assert rotated.status_code == 400 and "operator credential's hash" in rotated.json()["error"] diff --git a/tests/unit/test_router.py b/tests/unit/test_router.py index 9de51a1e75..84bd4db79d 100644 --- a/tests/unit/test_router.py +++ b/tests/unit/test_router.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager from typing import Any from unittest.mock import AsyncMock +from uuid import UUID import httpx import pytest @@ -46,6 +47,7 @@ def _row(**over: Any) -> directory.Cell: "next_fp": None, } values.update(over) + values["mutation_generation"] = UUID(int=1) return directory.Cell(**values) @@ -102,6 +104,8 @@ async def lookup(self, org: str) -> tuple[str, str] | None: async def set_limit(self, name: str, **fields: Any) -> directory.Cell: self._check() + expected = fields.pop("expected") + assert expected.generation == self.cells[name].mutation_generation given = {k: v for k, v in fields.items() if v is not directory.UNSET} self.cells[name] = dataclasses.replace(self.cells[name], **given) return self.cells[name] @@ -964,7 +968,13 @@ def test_a_cell_the_operator_closes_refuses_its_next_new_org( assert _place(client, "before").status_code == 200 closed = client.post( "/cells/v1/operator/cells/eu-1/limit", - json={"closed": True}, + json={ + "closed": True, + "expected": { + **{key: getattr(fake.cells["eu-1"], key, "") for key in directory.CONTROL_FIELDS}, + "mutation_generation": str(fake.cells["eu-1"].mutation_generation), + }, + }, headers={"Authorization": f"Bearer {operator}"}, ) assert closed.status_code == 200 and closed.json()["closed"] is True diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index efb0b29e80..1471f5882b 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -14,12 +14,14 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from uuid import UUID import httpx import pytest from starlette.testclient import TestClient from server.cli import cell as cell_cli +from server.cli import cell_control from server.core import cell from server.router import app as router_app from server.router import directory @@ -44,6 +46,7 @@ def _cell_row(**over: Any) -> dict[str, Any]: row: dict[str, Any] = { "cell": "eu-1", + "mutation_generation": UUID(int=1), "is_first": True, "host": "loom.example", "state": "attaching", @@ -102,7 +105,7 @@ def _row(self, name: str, sql: str) -> dict[str, Any] | None: async def execute(self, sql: str, *a: Any) -> str: self._count() cells = self.s.cells - if sql.startswith("SELECT pg_advisory_xact_lock"): + if sql.startswith(("SELECT pg_advisory_xact_lock", "SELECT pg_logical_emit_message")): return "" if "SET attach_started_at" in sql: row = cells[a[0]] @@ -172,10 +175,22 @@ async def fetch(self, sql: str, *a: Any) -> list[Any]: self.s.placements.update(dict.fromkeys(new, a[1])) return [{"org": o} for o in new] + def terminate(self) -> None: + pass + def transaction(self) -> Any: conn = self class _Tx: + async def start(self) -> None: + await self.__aenter__() + + async def commit(self) -> None: + await self.__aexit__(None) + + async def rollback(self) -> None: + await self.__aexit__(Exception) + async def __aenter__(self) -> None: conn._count() self.saved = copy.deepcopy( @@ -233,7 +248,9 @@ def listener() -> _Listener: @pytest.fixture def client(store: _Store, listener: _Listener) -> TestClient: router = router_app.Router(probe_transport=httpx.MockTransport(listener)) - return TestClient(router_app.build_app(router)) + client = TestClient(router_app.build_app(router)) + client.control_store = store + return client def _call( @@ -241,6 +258,11 @@ def _call( ) -> Any: url = "/cells/v1/operator/cells/eu-1" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {token}"} + if method != "GET" and (body is None or isinstance(body, dict)): + row = client.control_store.cells.get("eu-1", _cell_row()) + observed = {name: row[name] for name in directory.CONTROL_FIELDS} + observed["mutation_generation"] = str(row["mutation_generation"]) + body = {**(body or {}), "expected": observed} if body is None: return client.request(method, url, headers=headers) return client.request(method, url, json=body, headers=headers) @@ -249,6 +271,10 @@ def _call( def _begin(**over: Any) -> dict[str, Any]: body: dict[str, Any] = { "deployment_id": "dep-1", + "expected": { + **{name: _cell_row()[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, "current_fp": CURRENT, "next_fp": None, "keys_fp": "keys-1", @@ -292,6 +318,8 @@ def test_the_operator_reads_the_cells_row(client: TestClient, store: _Store) -> "current_fp": None, "next_fp": None, "other": None, + "mutation_generation": str(UUID(int=1)), + **{name: store.cells["eu-1"][name] for name in directory._REGISTERED}, } # Any other row is named: every attach call refuses while it exists. store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") @@ -322,6 +350,15 @@ def test_every_new_route_needs_the_operator_credential_and_names_an_unknown_cell assert store.statements == 0 url = "/cells/v1/operator/cells/eu-9" + (f"/{path}" if path else "") headers = {"Authorization": f"Bearer {OPERATOR}"} + if method != "GET": + source = _cell_row() + body = { + **(body or {}), + "expected": { + **{name: source[name] for name in directory.CONTROL_FIELDS}, + "mutation_generation": str(source["mutation_generation"]), + }, + } r = client.request(method, url, headers=headers, json=body) assert (r.status_code, r.json()["code"]) == (404, "NO_SUCH_CELL") bad = client.request(method, url.replace("eu-9", "EU_9"), headers=headers, json=body) @@ -613,7 +650,15 @@ def test_begin_refuses_a_fingerprint_whose_placement_body_would_pass_the_routers # And again under the lock. big = directory.Begin("dep-1", grown, None, "keys", {}) with pytest.raises(directory.OperatorRefused, match="configured_orgs"): - asyncio.run(directory.attach_begin("eu-1", big)) + asyncio.run( + directory.attach_begin( + "eu-1", + big, + expected=directory.Control( + {k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, UUID(int=1) + ), + ) + ) assert store.cells["eu-1"]["attach_started_at"] is None and store.audit == [] # One that fits, with the longest org escaped, is taken. fits = cell.as_value({**FIELDS, "configured_orgs": [f"org-{i:05d}" for i in range(100)]}) @@ -884,16 +929,33 @@ def sent(call: Any) -> int: asyncio.run(call) return store.statements - before + def observed(name: str = "eu-1") -> directory.Control: + row = store.cells[name] + return directory.Control( + {k: row[k] for k in directory.CONTROL_FIELDS}, row["mutation_generation"] + ) + begin = directory.Begin("dep-1", CURRENT, None, "keys", {}) assert sent(directory.attach_read("eu-1", audited=True)) == directory.OPERATOR_READ_STATEMENTS assert sent(directory.attach_read("eu-1")) == directory.ATTACH_READ_STATEMENTS - assert sent(directory.attach_begin("eu-1", begin)) == directory.ATTACH_CHANGE_STATEMENTS - assert sent(directory.attach_enforced("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS - batch = directory.attach_place("eu-1", "dep-1", [f"o{i}" for i in range(50)]) + assert ( + sent(directory.attach_begin("eu-1", begin, expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + assert ( + sent(directory.attach_enforced("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) + batch = directory.attach_place( + "eu-1", "dep-1", [f"o{i}" for i in range(50)], expected=observed() + ) assert sent(batch) == directory.ATTACH_PLACE_STATEMENTS # whatever the batch's size - assert sent(directory.attach_serving("eu-1", "dep-1")) == directory.ATTACH_CHANGE_STATEMENTS + assert ( + sent(directory.attach_serving("eu-1", "dep-1", expected=observed())) + == directory.ATTACH_CHANGE_STATEMENTS + ) store.cells["eu-2"] = _cell_row(cell="eu-2", is_first=False, host="eu-2.example") - assert sent(directory.remove("eu-2")) == directory.REMOVE_STATEMENTS + assert sent(directory.remove("eu-2", expected=observed("eu-2"))) == directory.REMOVE_STATEMENTS assert directory.REMOVE_MAX_S == directory._bound_s(directory.REMOVE_STATEMENTS) @@ -990,6 +1052,8 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: monkeypatch.setattr(settings, "router_operator_token", OPERATOR) fake = _Router( { + **_cell_row(), + "mutation_generation": str(UUID(int=1)), "cell": "eu-1", "is_first": True, "state": "attaching", @@ -999,6 +1063,23 @@ def cli_router(monkeypatch: pytest.MonkeyPatch) -> _Router: ) monkeypatch.setattr(httpx, "get", fake.get) monkeypatch.setattr(httpx, "delete", fake.delete) + + async def request( + url: Any, name: str, transport: Any, path: str, body: Any = None, *, method: str = "POST" + ) -> Any: + target = f"{url}/cells/v1/operator/cells/{name}" + path + if path == "/claim": + fake.calls.append(("POST", target)) + return dict(fake.row) + response = fake.get(target) if method == "GET" else fake.delete(target) + if response.status_code != 200: + if response.status_code == 404 and response.json().get("code") != "NO_SUCH_CELL": + raise cell_control._refused("a router replica is not upgraded") + raise cell_control._refused(response.json()["error"]) + return response.json() + + monkeypatch.setattr(cell_control, "request", request) + _stores(monkeypatch, "") return fake @@ -1007,13 +1088,24 @@ def _stores(monkeypatch: pytest.MonkeyPatch, refusal: Any) -> list[str]: it was asked about.""" checked: list[str] = [] - async def _check(deployment_id: str) -> str: - checked.append(deployment_id) + async def paired(_conn: Any, row: dict[str, Any]) -> None: + checked.append(row["deployment_id"]) if isinstance(refusal, Exception): raise refusal - return refusal + if refusal: + raise cell_control._refused(refusal) + + async def state(_conn: Any) -> tuple[str, str]: + return "open", "" + + async def held(_conn: Any) -> None: + pass - monkeypatch.setattr(cell_cli, "_first_cell_refusal", _check) + monkeypatch.setattr(cell_control, "paired", paired) + monkeypatch.setattr(cell_control, "session", lambda _what: contextlib.nullcontext(object())) + monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + monkeypatch.setattr(cell_control, "check_lock", held) + monkeypatch.setattr(cell, "read_state_if_present", state) return checked @@ -1022,9 +1114,9 @@ def test_remove_checks_a_first_cells_postgres_then_deletes_its_row( ) -> None: checked = _stores(monkeypatch, "") assert cell_cli._cmd_remove(_remove_args()) == 0 - assert checked == ["dep-1"] - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] - assert cli_router.calls[1][1] == "https://router.test/cells/v1/operator/cells/eu-1" + assert checked == ["dep-1", "dep-1"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] + assert cli_router.calls[-1][1] == "https://router.test/cells/v1/operator/cells/eu-1" assert json.loads(capsys.readouterr().out) == {"cell": "eu-1", "removed": True} @@ -1040,16 +1132,16 @@ def test_remove_refuses_a_first_cell_its_postgres_does_not_clear( ) -> None: _stores(monkeypatch, refusal) assert cell_cli._cmd_remove(_remove_args()) == 1 - assert "refusing" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET"] + assert "stop and rerun" in capsys.readouterr().err + assert [m for m, _url in cli_router.calls] == ["GET", "GET"] -def test_remove_reads_no_store_for_a_later_cell( +def test_remove_checks_a_later_cells_pairing_and_claim( cli_router: _Router, monkeypatch: pytest.MonkeyPatch ) -> None: cli_router.row = {**cli_router.row, "is_first": False, "deployment_id": "dep-2"} - checked = _stores(monkeypatch, "never asked") - assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == [] + checked = _stores(monkeypatch, "") + assert cell_cli._cmd_remove(_remove_args()) == 0 and checked == ["dep-2", "dep-2"] @pytest.mark.parametrize( @@ -1098,7 +1190,7 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( cli_router.deleted = (404, {"error": "the router answers no such path"}) assert cell_cli._cmd_remove(_remove_args()) == 1 assert "not upgraded" in capsys.readouterr().err - assert [m for m, _url in cli_router.calls] == ["GET", "DELETE"] + assert [m for m, _url in cli_router.calls] == ["GET", "GET", "POST", "DELETE"] @pytest.mark.parametrize( @@ -1112,42 +1204,31 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( ], ) def test_the_first_cell_check_needs_an_open_postgres_of_the_rows_deployment( - monkeypatch: pytest.MonkeyPatch, state: str, recorded: str, why: str + cli_router: _Router, + monkeypatch: pytest.MonkeyPatch, + capsys: Any, + state: str, + recorded: str, + why: str, ) -> None: - """One Postgres session of its own reads both: no pool, and no ClickHouse.""" - from server.core import pg - - sessions: list[dict[str, Any]] = [] - - class _Session: - async def fetchval(self, sql: str) -> str: - assert sql == "SELECT deployment_id FROM loom.authority" - return recorded + async def paired(_conn: Any, row: dict[str, Any]) -> None: + if not recorded: + raise cell_control._refused("records no deployment id") + if recorded != row["deployment_id"]: + raise cell_control._refused(f"records deployment id {recorded!r}") - @contextlib.asynccontextmanager - async def _dedicated(**kw: Any) -> Any: - sessions.append(kw) - yield _Session() - - async def _state(conn: Any) -> tuple[str, str]: - assert isinstance(conn, _Session) + async def read(_conn: Any) -> tuple[str, str]: return state, "" - def _no_pool(*_a: Any, **_kw: Any) -> None: - raise AssertionError("the check reads no pool") - - # LOOM_PG_DSN alone: the CLI's storage authority setting is not read. - monkeypatch.setattr(settings, "storage_authority", "clickhouse") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(pg, "dedicated", _dedicated) - monkeypatch.setattr(pg, "fetchrow", _no_pool) - monkeypatch.setattr(pg, "fetchval", _no_pool) - monkeypatch.setattr(cell, "read_state_if_present", _state) - refusal = asyncio.run(cell_cli._first_cell_refusal("dep-1")) - assert (why in refusal) if why else refusal == "" - assert sessions == [{"statement_timeout_ms": cell_cli._CHECK_STATEMENT_MS}] - monkeypatch.setattr(settings, "pg_dsn", "") - assert "LOOM_PG_DSN" in asyncio.run(cell_cli._first_cell_refusal("dep-1")) + monkeypatch.setattr(cell_control, "paired", paired) + monkeypatch.setattr(cell, "read_state_if_present", read) + result = cell_cli._cmd_remove(_remove_args()) + if why: + assert result == 1 + assert why.replace("`", "") in capsys.readouterr().err + assert not any(method in ("POST", "DELETE") for method, _url in cli_router.calls) + else: + assert result == 0 def test_remove_never_connects_clickhouse( @@ -1158,13 +1239,7 @@ def test_remove_never_connects_clickhouse( def _never(*_a: Any, **_kw: Any) -> None: raise AssertionError("loom cell remove connects no ClickHouse") - async def _read() -> tuple[str, str]: - return "open", "dep-1" - monkeypatch.setattr(_ch, "run_connected", _never) - monkeypatch.setattr(settings, "storage_authority", "postgres") - monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") - monkeypatch.setattr(cell_cli, "_read_first_cell", _read) assert cell_cli._cmd_remove(_remove_args()) == 0 @@ -1211,7 +1286,7 @@ def test_a_rotation_drops_this_replicas_cached_state( store.cells["eu-1"]["state"] = "serving" assert _state(client).json()["state"] == "attaching" # cached - async def _rotate(name: str, step: str, sha: str) -> directory.Cell: + async def _rotate(name: str, step: str, sha: str, **_kw: Any) -> directory.Cell: store.cells[name]["credential_next_sha256"] = sha return directory._cell(_Conn(store)._row(name, "")) @@ -1303,13 +1378,14 @@ def test_attach_serving_waits_for_a_state_read_in_flight_and_drops_its_row( after attach/serving runs once that read is done, so the next read answers `serving`.""" router = router_app.Router(probe_transport=httpx.MockTransport(listener)) with TestClient(router_app.build_app(router)) as setup: + setup.control_store = store _enforced(setup) started, go = _blocked_state_read(store, monkeypatch) committed = asyncio.Event() real_serving = directory.attach_serving - async def _serving(*a: Any) -> Any: - found = await real_serving(*a) + async def _serving(*a: Any, **kw: Any) -> Any: + found = await real_serving(*a, **kw) committed.set() return found @@ -1325,7 +1401,14 @@ async def _run() -> None: serving = asyncio.create_task( c.post( "/cells/v1/operator/cells/eu-1/attach/serving", - json={"deployment_id": "dep-1", "nonces": NONCES}, + json={ + "deployment_id": "dep-1", + "nonces": NONCES, + "expected": { + **{k: store.cells["eu-1"][k] for k in directory.CONTROL_FIELDS}, + "mutation_generation": str(UUID(int=1)), + }, + }, headers={"Authorization": f"Bearer {OPERATOR}"}, ) ) From e0f756bb5c6676002bb8d2740e880d9725fe727e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:13:29 +0000 Subject: [PATCH 038/181] Index cell control recovery and its upgrade refusal Signed-off-by: zlareb1 --- docs/reference/errors.md | 4 ++++ llms.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 62a4a86f46..a2670a9c72 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -432,6 +432,10 @@ Raised in `server/memory/domain/record_values.py`. Raised in `server/tracker/app/sources.py`. +#### `CONTROL_UPGRADE_REQUIRED` + +Raised in `server/router/app.py`. + #### `CREDENTIAL_NOT_READONLY` Raised in `server/tracker/app/sources.py`. diff --git a/llms.txt b/llms.txt index 43b8421b0e..391bbc9c1d 100644 --- a/llms.txt +++ b/llms.txt @@ -105,6 +105,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Exporting usage to a data warehouse](docs/operations/analytics-export.md): `loom export-analytics` writes one UTC day of usage and a snapshot of the namespaces to an S3 bucket as Parquet files, one file per entity per day, for a data warehouse to load. - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. - [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… +- [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). From 3f62dfdc92cda9e061b36444e541f438b3102916 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:15:54 +0000 Subject: [PATCH 039/181] Validate the cell name before recovery requests Signed-off-by: zlareb1 --- docs/operations/cell-control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index ccd7bf5edf..2ef90c33a7 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -82,7 +82,7 @@ async def main(): if not recorded or recorded["authority"] != "postgres": raise RuntimeError("the configured stores do not record Postgres authority") pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) - url = cell_cli.operator_router(argparse.Namespace(router=None)) + url = cell_cli._cell_router(argparse.Namespace(router=None, cell=settings.cell)) if not isinstance(url, httpx.URL): raise RuntimeError("the router URL or operator credential was refused") From db89cb2058cf9d8791c537a166c3335473842f69 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:53:14 +0000 Subject: [PATCH 040/181] Keep attaching cells closed through state cleanup Refuse explicit open requests under the guarded attaching row lock, preserving quota atomicity and the durable zero-placement boundary. Inventory control writers and their required boundaries, including dynamic cleanup SQL. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 +- docs/operations/cell-control.md | 14 +- server/cli/cell.py | 2 +- server/router/directory.py | 4 + tests/integration/test_cell_control_fence.py | 188 +++++++ tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_control_census.py | 503 +++++++++++++++++++ 8 files changed, 716 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cell_control_census.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index e5419326ed..59ed6fef64 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1203,7 +1203,10 @@ one command, `loom cell limit --cell [--closed | --open] [--placements-max ( | none)]`, which writes the directory alone, with the router's operator credential and none of the cell's stores, so it works while the cell is down; the router judges a placement from -the directory row. +the directory row. An explicit `--open` refuses while the cell is `attaching`, including +an already-open row and a combined quota edit. This behavior break preserves the closed, +zero-placement boundary used by state cleanup. Close and quota-only edits still work; +an attaching cell can complete attach while closed, then reopen once `serving`. ## Control mutation fencing @@ -1245,7 +1248,9 @@ cleanup have explicit bounds; an interrupted or uncertain local transaction neve continues on another connection. No operation record, persistent census, phase or lease is added. The UUID is coordination metadata and holds no memory content. -The source census for acceptance and operator directory writes is: +The source census for acceptance and operator directory writes is checked by +`tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded +entry points and the named exceptions: | Writer | Fence | | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | @@ -1274,7 +1279,7 @@ participate in this protocol. there); for an org with no placement it names the first cell, which the lookup does not tell apart. - `loom cell limit --cell --closed` closes a cell to new orgs, `--open` reopens - it, and `--placements-max ` sets its placement quota (`--placements-max none` + a serving cell, and `--placements-max ` sets its placement quota (`--placements-max none` clears it). - `loom cell remove --cell ` removes the directory row of a cell that is `attaching` with no placement (`--cell` is the cell's `LOOM_CELL`, unset by then). For the first cell diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index 2ef90c33a7..b606d9aaf9 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -102,15 +102,21 @@ asyncio.run(main()) The helper claims first, closes the cell using the existing limit operation, and requires that operation's durable placement count to remain zero before any local state cleanup. A queued placement either wins first and refuses cleanup, or observes the closed row and -cannot create a placement. The helper then compares and deletes only the existing +cannot create a placement. An explicit `loom cell limit --open` now refuses for every +attaching cell, even one already open; a combined open/quota request changes neither +field. This behavior break keeps concurrent limit clients from reopening the cleanup +boundary. Close and quota-only edits remain available. The helper then compares and +deletes only the existing `cell_first`, `tenancy_mode`, `cell_self` and `tenancy` rows, restores the first-row triggers and drops only the existing org guard triggers, on the same locked backend. It takes the migration lock for each bounded local transaction. Even an already-open retry proves durability before reporting success. -An abort leaves the cell closed. It never reopens automatically or deletes a placement. -If a placement won, resume attach; inspect the reason before explicitly reopening with -`loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run +After the close is acknowledged, an abort leaves the cell closed. It never reopens +automatically or deletes a placement. +If a placement won, resume attach while closed: its existing-org placement batches bypass +the ordinary close/quota gate. Complete attach to `serving` before explicitly reopening +with `loom cell limit --open`. Otherwise rerun the cleanup with a fresh claim, then run `loom cell remove --cell ` with the same cell settings. Removal now holds that cell's direct Postgres lock and takes a fresh claim for both first and later cells. It also requires a first cell's durable local state to read `open` before deleting the row. diff --git a/server/cli/cell.py b/server/cli/cell.py index 6d6f23697f..77b13e8aec 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -439,7 +439,7 @@ def _add_cell_parser(sub: Any) -> None: limit.add_argument("--cell", required=True, help="The cell to limit") state = limit.add_mutually_exclusive_group() state.add_argument("--closed", action="store_true", help="Refuse its new orgs") - state.add_argument("--open", action="store_true", help="Take new orgs again") + state.add_argument("--open", action="store_true", help="Take new orgs again once serving") limit.add_argument( "--placements-max", metavar="N|none", help="Its quota of orgs (none: no quota)" ) diff --git a/server/router/directory.py b/server/router/directory.py index 185c5126d6..5158d24cf7 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -314,6 +314,10 @@ async def set_limit( raise ValueError("set_limit sets closed, placements_max or both") async def change(conn: Any, row: Cell) -> dict[str, Any]: + if closed is False and row.state == "attaching": + raise OperatorRefused( + f"cell {cell_name!r} is attaching: it can reopen after attach reaches serving" + ) detail: dict[str, Any] = {} if closed is not UNSET: detail["closed"] = closed diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index a6799d2064..d96f170fa3 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -144,6 +144,7 @@ async def delayed(*args, **kwargs): @pytest.mark.asyncio async def test_replayed_claim_lost_ack_and_noop_wrong_generation(control_stores) -> None: client, conn, _ = control_stores + await conn.execute("UPDATE loom_router.cells SET state = 'serving'") snapshot = await read(client) lost = await claim(client, snapshot) # its caller lost this acknowledgement current = await claim(client, snapshot) # duplicate HTTP execution must rotate again @@ -611,3 +612,190 @@ async def ready(): finally: await a.close() await b.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fresh_open", [False, True]) +async def test_backout_close_cannot_be_reopened_before_same_session_cleanup( + control_stores, local_control, fresh_open +): + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, _ = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode = 'enforced', placements_max = 7" + ) + await local.execute("INSERT INTO loom.cell_first(is_first, cell) VALUES (true, 'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES ('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES ('{\"v\":1}')") + await apg.take_lock(local, "backout reopen test") + pid = local.get_server_pid() + before_close = opened = placement_error = None + + async def call(path, body): + nonlocal before_close, opened, placement_error + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + answer = response.json() + if path == "/claim": + # The queued opener has the NEW generation, read before cleanup closes D. + before_close = await read(client) + if path == "/limit": + assert answer["closed"] and answer["placed_orgs"] == 0 + assert await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert await apg.lock_held(local) + snapshot = await read(client) if fresh_open else before_close + opened = await client.post( + "/cells/v1/operator/cells/first/limit", + json={**expected(snapshot), "closed": False, "placements_max": 99}, + ) + # Actually attempt placement before the acknowledged close returns to cleanup. + try: + await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + except directory.PlacementRefused as exc: + placement_error = str(exc) + return answer + + await cell_control.backout(local, call, await read(client)) + assert opened.status_code == 409, opened.text + assert "serving" in opened.json()["error"] + assert placement_error and "closed" in placement_error + after = await read(client) + assert after["placed_orgs"] == 0 + assert tuple( + await directory_admin.fetchrow("SELECT closed, placements_max FROM loom_router.cells") + ) == (True, 7) + assert after["mutation_generation"] == before_close["mutation_generation"] + assert not await directory_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom_router.placements)") + assert ( + await directory_admin.fetchval( + "SELECT count(*) FROM loom_router.audit WHERE action = 'limit'" + ) + == 1 + ) + assert local.get_server_pid() == pid and await apg.lock_held(local) + for table in ("cell_first", "tenancy_mode", "cell_self", "tenancy"): + assert not await local.fetchval(f"SELECT EXISTS(SELECT 1 FROM loom.{table})") + + +@pytest.mark.asyncio +async def test_attaching_open_refusal_preserves_guard_order_and_closed_attach_progress( + control_stores, +): + client, conn, _ = control_stores + original = await read(client) + current = await claim(client, original) + + async def limit(snapshot, **fields): + return await client.post( + "/cells/v1/operator/cells/first/limit", json={**expected(snapshot), **fields} + ) + + stale = await limit(original, closed=False, placements_max=99) + assert stale.status_code == 409 and "generation" in stale.json()["error"] + for field, value in (("host", "other.example.test"), ("current_fp", {"v": 2})): + changed = await limit({**current, field: value}, closed=False, placements_max=99) + assert ( + changed.status_code == 409 and "registration or fingerprints" in changed.json()["error"] + ) + noop = await limit(current, closed=False, placements_max=99) + assert noop.status_code == 409 and "serving" in noop.json()["error"] + assert await conn.fetchval("SELECT placements_max FROM loom_router.cells") is None + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.audit WHERE action='limit')" + ) + assert (await limit(current, closed=True)).status_code == 200 + assert (await limit(current, placements_max=0)).status_code == 200 + closed = await read(client) + assert tuple(await conn.fetchrow("SELECT closed, placements_max FROM loom_router.cells")) == ( + True, + 0, + ) + assert closed["mutation_generation"] == current["mutation_generation"] + + # An earlier mode request cannot escape the cleanup owner's claim. + stale_mode = await client.post( + "/cells/v1/operator/cells/first/attach/enforced", + json={**expected(original), "deployment_id": "dep-first"}, + ) + assert stale_mode.status_code == 409 and "generation" in stale_mode.json()["error"] + # Existing held-org attach batches intentionally bypass closed/quota. + await conn.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + row = await read(client) + values = expected(row, generation=False)["expected"] + control = directory.Control(values, uuid.UUID(row["mutation_generation"])) + assert await directory.attach_place("first", "dep-first", ["held-org"], expected=control) == ( + 1, + 1, + ) + served = await directory.attach_serving("first", "dep-first", expected=control) + assert served.row.closed and served.row.state == "serving" + response = await limit(await read(client), closed=False) + assert response.status_code == 200, response.text + assert not response.json()["closed"] + + +@pytest.mark.asyncio +async def test_pending_attach_batch_cannot_escape_backout_claim( + control_stores, local_control, monkeypatch +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from server.cli import cell_control + + client, directory_admin, _ = control_stores + local, dsn = local_control + await directory_admin.execute( + "UPDATE loom_router.cells SET mode='enforced', attach_started_at=now(), current_fp='{\"v\":1}'" + ) + await local.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await local.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + await local.execute("INSERT INTO loom.cell_self(current_fp) VALUES('{\"v\":1}')") + old_backend = await asyncpg.connect(dsn) + await apg.take_lock(old_backend, "old attach batch") + old = await claim(client, await read(client)) + entered, resume = asyncio.Event(), asyncio.Event() + real = directory.attach_place + + async def delayed(*args, **kwargs): + entered.set() + await resume.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(directory, "attach_place", delayed) + pending = asyncio.create_task( + client.post( + "/cells/v1/operator/cells/first/attach/place", + json={**expected(old), "deployment_id": "dep-first", "orgs": ["old-batch"]}, + ) + ) + + async def call(path, body): + response = await client.post("/cells/v1/operator/cells/first" + path, json=body) + assert response.status_code == 200, response.text + if path == "/limit": + resume.set() + stale = await pending + assert stale.status_code == 409 and "generation" in stale.json()["error"] + return response.json() + + try: + await asyncio.wait_for(entered.wait(), 3) + await local.execute("SELECT pg_terminate_backend($1)", old_backend.get_server_pid()) + await apg.take_lock(local, "backout successor") + await cell_control.backout(local, call, await read(client)) + assert await apg.lock_held(local) + assert not await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") + assert not await directory_admin.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.placements)" + ) + assert await directory_admin.fetchval("SELECT closed FROM loom_router.cells") + finally: + resume.set() + await pending + await old_backend.close() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 4e45db4285..1e7acd696a 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -445,7 +445,7 @@ ], "loom cell limit": [ "usage: loom cell limit [-h] --cell CELL [--closed | --open] [--placements-max N|none] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "options: -h, --help show this help message and exit --cell CELL The cell to limit --closed Refuse its new orgs --open Take new orgs again once serving --placements-max N|none Its quota of orgs (none: no quota) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom cell which": [ "usage: loom cell which [-h] --org ORG [--router ROUTER]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index a31a862106..3bc4f24ee5 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12622,7 +12622,7 @@ "nargs": 0, "type": null, "required": false, - "help": "Take new orgs again", + "help": "Take new orgs again once serving", "metavar": null }, { diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py new file mode 100644 index 0000000000..aec52c908c --- /dev/null +++ b/tests/unit/test_cell_control_census.py @@ -0,0 +1,503 @@ +"""Inventory cell control SQL writers and the shared boundaries they must retain. + +This recognizes Python string literals (including adjacent literals) and f-strings. It is a +source change gate, not a proof for arbitrary generated SQL; backend/session behavior +is exercised against Postgres in test_cell_control_fence. +""" + +from __future__ import annotations + +import ast +import re +from collections import Counter +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +# Include immutable placement writes: their exemption must remain explicit. +TABLES = r"(?:loom\.(?:cell_self|cell_first|tenancy_mode)|loom_router\.[A-Za-z_]+)" +TARGET = rf"(?:{TABLES}|loom(?:_router)?\.\{{[^}}]+\}}|\{{[^}}]+\}})" +WRITES = re.compile( + rf"(?:^|;)\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" + rf"ALTER\s+TABLE(?:\s+IF\s+EXISTS)?|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" + rf"DROP\s+TABLE(?:\s+IF\s+EXISTS)?)\s+({TARGET})" + rf"|(?:^|;)\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", + re.IGNORECASE, +) + + +def mutations(sql: str) -> list[tuple[str, str]]: + sql = re.sub(r"--[^\n]*", "", sql).replace('"', "") + return [(" ".join((a or c).upper().split()), b or d) for a, b, c, d in WRITES.findall(sql)] + + +class Inventory(ast.NodeVisitor): + def __init__(self) -> None: + self.scope: list[str] = [] + self.writes: Counter[tuple[str, str, str]] = Counter() + self.functions: dict[str, ast.AST] = {} + + def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None: + self.scope.append(node.name) + self.functions[".".join(self.scope)] = node + self.generic_visit(node) + self.scope.pop() + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self.scope.append(node.name) + self.generic_visit(node) + self.scope.pop() + + def record(self, sql: str) -> None: + for verb, target in mutations(sql): + self.writes[(".".join(self.scope), verb, target)] += 1 + + def visit_Constant(self, node: ast.Constant) -> None: + if isinstance(node.value, str): + self.record(node.value) + + def visit_JoinedStr(self, node: ast.JoinedStr) -> None: + self.record( + "".join( + str(part.value) + if isinstance(part, ast.Constant) + else "{" + ast.unparse(part.value) + "}" + for part in node.values + ) + ) + # Do not count each f-string fragment as another SQL statement. + + +def inventory(source: str) -> Inventory: + found = Inventory() + found.visit(ast.parse(source)) + return found + + +def test_repository_cell_control_writers_are_classified(): + actual = {} + for path in (ROOT / "server").rglob("*.py"): + writes = inventory(path.read_text()).writes + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + expected = {} + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS): + for path, entries in group.items(): + expected.setdefault(path, {}).update(entries) + assert actual == expected + + +def test_cell_control_migrations_are_classified(): + actual = {} + for path in (ROOT / "schemas").rglob("*.sql"): + writes = Counter(mutations(path.read_text())) + if writes: + actual[str(path.relative_to(ROOT))] = dict(writes) + assert actual == MIGRATION_WRITERS + + +# Entries name every sink, including nested callbacks and dynamic table expressions. +# A new sink, including another statement in an existing owner, requires classification. +GUARDED_WRITERS = { + "server/cli/cell_attach_pg.py": {("_mode_try", "UPDATE", "loom.tenancy_mode"): 1}, + "server/cli/cell_control.py": { + ("backout", "ALTER TABLE", "loom.cell_first"): 3, + ("backout", "DELETE FROM", "loom.{table}"): 1, + ("backout", "DROP TRIGGER", "loom.{table}"): 1, + }, + "server/cli/cell_prepare.py": { + ("_write_rows", "INSERT INTO", "loom.cell_first"): 1, + ("_write_rows", "INSERT INTO", "loom.tenancy_mode"): 1, + ("_write_rows", "INSERT INTO", "loom.cell_self"): 1, + }, + "server/router/directory.py": { + ("set_limit.change", "UPDATE", "loom_router.cells"): 1, + ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, + ("claim", "UPDATE", "loom_router.cells"): 1, + ("attach_begin", "UPDATE", "loom_router.cells"): 1, + ("attach_place", "INSERT INTO", "loom_router.placements"): 1, + ("attach_place", "UPDATE", "loom_router.cells"): 1, + ("attach_place", "INSERT INTO", "loom_router.audit"): 1, + ("attach_enforced", "UPDATE", "loom_router.cells"): 1, + ("attach_serving", "UPDATE", "loom_router.cells"): 1, + ("remove", "DELETE FROM", "loom_router.hosts"): 1, + ("remove", "DELETE FROM", "loom_router.cells"): 1, + }, +} + +# Registration only bootstraps or validates its immutable rerun. Ordinary placement +# writes an immutable org placement/count under its org and cell locks. Reads write +# audit rows only. Trigger setup only enables/installs guards under the migration lock. +EXEMPT_WRITERS = { + "server/core/cell_triggers.py": { + ("_create", "CREATE TRIGGER", "{qualified}"): 2, + ("ensure", "ALTER TABLE", "loom.cell_first"): 1, + ("_table_statements", "ALTER TABLE", "loom.{_ident(table)}"): 1, + }, + "server/router/directory.py": { + ("register", "INSERT INTO", "loom_router.cells"): 1, + ("register", "INSERT INTO", "loom_router.hosts"): 1, + ("register", "INSERT INTO", "loom_router.audit"): 1, + ("_place_one", "INSERT INTO", "loom_router.placements"): 1, + ("_place_one", "UPDATE", "loom_router.cells"): 1, + ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("_audit", "INSERT INTO", "loom_router.audit"): 1, + }, +} + +# Unresolved table SQL outside cell controls is classified by exact function and +# target expression, never by a whole-file or execute/f-string exemption. These are +# ClickHouse content/retention/rebuild stores, Keeper locks/marks and provider tables; +# pg_migrate writes only its named migration ledger, and prepare's nonce writes only +# the probe lock table. None is a cell acceptance or directory control write. +GENERIC_SQL_EXCLUSIONS = { + "server/cli/cell_prepare.py": {("_write_nonces", "INSERT INTO", "{leases.LOCK_TABLE}"): 1}, + "server/code/store/index/prefix_twin.py": {("fill_sql", "INSERT INTO", "{table}"): 1}, + "server/core/ch.py": {("_fenced_statement", "INSERT INTO", "{qualify(table)}"): 1}, + "server/core/leases.py": { + ("", "INSERT INTO", "{LOCK_TABLE}"): 1, + ("", "ALTER TABLE", "{LOCK_TABLE}"): 4, + }, + "server/core/namespace_write_marks.py": {("", "INSERT INTO", "{KEYS_TABLE}"): 1}, + "server/core/schema/migrate_lease.py": {("", "CREATE TABLE IF NOT EXISTS", "{LOCK_TABLE}"): 1}, + "server/core/schema/pg_migrate.py": { + ("_ledger", "CREATE TABLE IF NOT EXISTS", "{schema}"): 1, + ("_record_online", "INSERT INTO", "{schema}"): 1, + ("_apply", "INSERT INTO", "{schema}"): 1, + }, + "server/core/schema/table_rebuild.py": { + ("copy_sql", "INSERT INTO", "loom.{dst}"): 1, + ("apply", "DROP TABLE", "loom.{twin}"): 2, + }, + "server/erasure/store/mutations.py": { + ("run_mutation", "ALTER TABLE", "{table}"): 1, + ("run_lightweight_delete", "DELETE FROM", "{table}"): 1, + ("run_keeper_delete", "ALTER TABLE", "{table}"): 1, + }, + "server/erasure/store/reclaim.py": {("apply_deleted_mask", "ALTER TABLE", "{table}"): 1}, + "server/erasure/store/steps_memory.py": { + ("_scrub_link_cursors", "DELETE FROM", "{_CURSORS}"): 1 + }, + "server/erasure/store/steps_principals.py": {("_rewrite", "INSERT INTO", "{table}"): 1}, + "server/insights/domain/analysis.py": {("entity_sql", "INSERT INTO", "{db}"): 1}, + "server/insights/domain/provider_prepared_ddl.py": { + ("prepared_ddl", "CREATE TABLE", "loom.{prepared}"): 1 + }, + "server/insights/store/provider_cache.py": { + ("invalidate_staged_cache", "TRUNCATE TABLE", "loom.{names.target}"): 1 + }, + "server/insights/store/provider_cleanup.py": { + ("cleanup_rebuild_copies", "DROP TABLE IF EXISTS", "{db}"): 1 + }, + "server/insights/store/provider_participants.py": {("", "ALTER TABLE", "{TABLE}"): 1}, + "server/insights/store/provider_prefix.py": { + ("derive_staged_prefixes", "TRUNCATE TABLE", "loom.{names.target}"): 1, + ("derive_staged_prefixes", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/insights/store/provider_prepared.py": { + ("prepare_live_table", "DROP TABLE", "loom.{names.prepared}"): 1, + ("prepare_live_table", "INSERT INTO", "loom.{names.prepared}"): 1, + ("prepare_live_table", "ALTER TABLE", "loom.{names.prepared}"): 1, + }, + "server/insights/store/provider_staging.py": { + ("create_snapshot", "CREATE TABLE", "loom.{names.snapshot}"): 1, + ("complete_snapshot", "ALTER TABLE", "loom.{names.snapshot}"): 1, + ("discard_snapshot", "DROP TABLE", "loom.{names.snapshot}"): 1, + ("snapshot_query_running", "CREATE TABLE", "loom.{names.snapshot}"): 1, + }, + "server/insights/store/provider_target.py": { + ("ensure_target", "CREATE TABLE", "loom.{names.target}"): 1, + ("_insert_patches", "INSERT INTO", "loom.{names.target}"): 1, + }, + "server/knowledge/store/blobs.py": { + ("copy_held", "INSERT INTO", "{TABLE}"): 1, + ("unwrite", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{TABLE}"): 1, + ("delete", "DELETE FROM", "{UPLOADS}"): 1, + ("forget_uploads", "DELETE FROM", "{UPLOADS}"): 1, + }, + "server/knowledge/store/chunks.py": { + ("_record_supersessions", "INSERT INTO", "{SUPERSESSIONS_TABLE}"): 1 + }, + "server/knowledge/store/conversations.py": { + ("mark_message_saved", "ALTER TABLE", "{table}"): 1, + ("unsave_message", "DELETE FROM", "{table}"): 1, + ("unsave_message", "ALTER TABLE", "{table}"): 1, + }, + "server/knowledge/store/datasets.py": { + ("forget", "DELETE FROM", "{ROWS}"): 1, + ("forget", "DELETE FROM", "{DATASETS}"): 1, + ("forget_published", "DELETE FROM", "{ROWS}"): 1, + ("forget_published", "DELETE FROM", "{DATASETS}"): 1, + ("copy_snapshot", "INSERT INTO", "{ROWS}"): 1, + ("forget_snapshot", "DELETE FROM", "{ROWS}"): 1, + }, + "server/knowledge/store/gaps.py": { + ("delete_miss", "DELETE FROM", "{MISSES}"): 1, + ("delete_build", "DELETE FROM", "{GAPS}"): 1, + ("delete_misses", "DELETE FROM", "{MISSES}"): 1, + ("delete_builds", "DELETE FROM", "{GAPS}"): 1, + }, + "server/knowledge/store/restore.py": {("unretire", "DELETE FROM", "{SUPERSESSIONS_TABLE}"): 1}, + "server/memory/app/reembed.py": { + ("in_database_sql", "INSERT INTO", "loom.{table}"): 1, + ("delete_off_model", "DELETE FROM", "loom.{table}"): 1, + }, + "server/memory/store/code_links.py": { + ("_purge_tombstones", "DELETE FROM", "{TABLE}"): 1, + ("_drop_gone", "DELETE FROM", "{TABLE}"): 1, + }, + "server/memory/store/graph_projections.py": {("KeeperLayout", "INSERT INTO", "{TABLE}"): 1}, + "server/memory/store/lifecycle.py": {("_tombstone_where", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/redaction.py": {("delete_by_key", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/term_freq.py": {("rebuild", "ALTER TABLE", "{table}"): 1}, + "server/memory/store/webhooks.py": {("mark_dead", "INSERT INTO", "{ATTEMPTS}"): 1}, + "server/memory/workers/usage_fold.py": { + ("_apply_update", "UPDATE", "{table}"): 1, + ("_apply_update", "ALTER TABLE", "{table}"): 1, + }, + "server/storage/retention_overrides.py": { + ("compute_alters", "ALTER TABLE", "{target.table}"): 1, + ("current_ttl_matches", "ALTER TABLE", "{table}"): 1, + ("apply", "ALTER TABLE", "{target.table}"): 1, + }, +} + +# Schema creation/extension and immutable-trigger installation run as migrations, +# never as live acceptance writers. Offline restore generation rotation stays stopped +# and fenced as described in the recovery guide, not a live SQL exemption. +MIGRATION_WRITERS = { + "schemas/pg_migrations/008_cells.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom.cell_first"): 1, + ("DROP TRIGGER", "loom.cell_first"): 2, + ("CREATE TRIGGER", "loom.cell_first"): 2, + ("ALTER TABLE", "loom.cell_first"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom.tenancy_mode"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom.cell_self"): 1, + }, + "schemas/router_migrations/001_directory.sql": { + ("CREATE TABLE IF NOT EXISTS", "loom_router.cells"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.placements"): 1, + ("DROP TRIGGER", "loom_router.placements"): 2, + ("CREATE TRIGGER", "loom_router.placements"): 2, + ("CREATE TABLE IF NOT EXISTS", "loom_router.hosts"): 1, + ("CREATE TABLE IF NOT EXISTS", "loom_router.audit"): 1, + }, + "schemas/router_migrations/002_cell_upstream.sql": {("ALTER TABLE", "loom_router.cells"): 1}, + "schemas/router_migrations/003_cell_next_credential.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, + "schemas/router_migrations/005_control_generation.sql": { + ("ALTER TABLE", "loom_router.cells"): 1 + }, +} + + +def calls(node: ast.AST) -> set[str]: + return {ast.unparse(child) for child in ast.walk(node) if isinstance(child, ast.Call)} + + +def function(path: str, name: str) -> ast.AST: + return inventory((ROOT / path).read_text()).functions[name] + + +@pytest.mark.parametrize( + ("path", "name", "required"), + [ + ("server/cli/cell_prepare.py", "attach_lock", ['cell_control.session("prepare")']), + ( + "server/cli/cell_prepare.py", + "_prepare_locked", + [ + "cell_control.claim(conn, call, snapshot)", + "_write_rows(conn, rows, answer, current, nxt)", + ], + ), + ( + "server/cli/cell_prepare.py", + "_write_rows", + ["cell_control.transaction(conn)", "_read_rows(conn, count_tenancy=False)"], + ), + ( + "server/cli/cell_attach.py", + "Run.run", + ["self.open_lock()", "cell_control.claim(self.lock_conn, self.router.call, self.row)"], + ), + ( + "server/cli/cell_attach.py", + "Run.commit_mode", + [ + "apg.commit_enforced(self.lock_conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" + ], + ), + ( + "server/cli/cell_attach_pg.py", + "_mode_try", + ["cell_control.check_lock(conn)", "conn.transaction()", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "transaction", + ["check_lock(conn)", "tx.commit()"], + ), + ( + "server/cli/cell_control.py", + "claim", + ["check_lock(conn)", "paired(conn, row)"], + ), + ( + "server/cli/cell_control.py", + "backout", + [ + "claim(conn, call, row)", + 'mutate(conn, call, row, "/limit", {"closed": True})', + "transaction(conn)", + "paired(conn, row)", + ], + ), + ( + "server/cli/cell.py", + "_remove_locked", + [ + 'cell_control.session("remove")', + "cell_control.claim(conn, call, row)", + "cell_control.check_lock(conn)", + ], + ), + ( + "server/router/directory.py", + "_attach_change", + ["_control_transaction()", "control_refusal(attaching, expected, claim=claim)"], + ), + ( + "server/router/directory.py", + "_operator_change", + ["_attach_change(cell_name, expected)", "change(conn, found.row)"], + ), + ( + "server/router/directory.py", + "set_limit", + ['_operator_change(cell_name, "limit", change, expected)'], + ), + ( + "server/router/directory.py", + "rotate_credential", + ['_operator_change(cell_name, "credential", change, expected)'], + ), + ( + "server/router/directory.py", + "claim", + ["_attach_change(cell_name, expected, claim=True)"], + ), + *[ + ("server/router/directory.py", name, ["_attach_change(cell_name, expected)"]) + for name in ( + "attach_begin", + "attach_place", + "attach_enforced", + "attach_serving", + "remove", + ) + ], + ], +) +def test_guarded_entrypoints_keep_shared_boundaries(path, name, required): + actual = calls(function(path, name)) + for expression in required: + assert ast.unparse(ast.parse(expression, mode="eval").body) in actual, ( + path, + name, + expression, + ) + + +def test_dynamic_cleanup_scope_cannot_expand_unclassified(): + backout = function("server/cli/cell_control.py", "backout") + cleanup_tables = [ + ast.literal_eval(node.iter) + for node in ast.walk(backout) + if isinstance(node, ast.For) + and ast.unparse(node.target) == "table" + and isinstance(node.iter, ast.Tuple) + ] + assert cleanup_tables == [("cell_first", "tenancy_mode", "cell_self", "tenancy")] + assert "t.tgname IN ('tenancy_insert', 'tenancy_update', 'tenancy_record')" in ast.unparse( + backout + ) + + +def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): + path = "server/router/directory.py" + found = inventory((ROOT / path).read_text()) + for name in ( + "routing", + "credentials", + "first_cell", + "first_cell_named", + "lookup", + "attach_read", + ): + assert name in found.functions + assert not any(owner == name for owner, _verb, _target in found.writes) + # Operator reads may audit, but cannot bypass control guards for a state write. + assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) + assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) + assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( + found.functions["place"] + ) + assert "refusal(candidate)" in calls(found.functions["_place_one"]) + for name in ("set_limit", "rotate_credential"): + assert not any(call.startswith("claim(") for call in calls(found.functions[name])) + + +@pytest.mark.parametrize( + ("statement", "expected"), + [ + ('"UPDATE loom.cell_self SET current_fp = $1"', ("UPDATE", "loom.cell_self")), + ( + '"INSERT INTO loom_router.cells (cell) VALUES ($1)"', + ("INSERT INTO", "loom_router.cells"), + ), + ('f"DELETE FROM loom.{table}"', ("DELETE FROM", "loom.{table}")), + ('f"UPDATE {target} SET current_fp = $1"', ("UPDATE", "{target}")), + ('"DROP TABLE IF EXISTS loom.cell_self"', ("DROP TABLE IF EXISTS", "loom.cell_self")), + ('f\'DROP TRIGGER "{name}" ON loom."{table}"\'', ("DROP TRIGGER", "loom.{table}")), + ], +) +def test_inventory_detects_new_literal_and_dynamic_writers(statement, expected): + source = ( + f"async def unclassified(conn, table, target, name):\n await conn.execute({statement})\n" + ) + found = inventory(source).writes + assert found == Counter({("unclassified", *expected): 1}) + assert not any( + "unclassified" == owner + for group in (GUARDED_WRITERS, EXEMPT_WRITERS, GENERIC_SQL_EXCLUSIONS) + for entries in group.values() + for owner, _verb, _target in entries + ) + + +def test_acceptance_writer_callsites_are_classified(): + # Another caller of a checked writer still needs its own claim and ownership proof. + names = {"_write_rows", "_mode_try", "commit_enforced", "backout"} + actual = Counter() + for path in (ROOT / "server").rglob("*.py"): + found = inventory(path.read_text()) + for owner, node in found.functions.items(): + # These owners have no nested functions; walk only the direct callsites. + for child in ast.walk(node): + if isinstance(child, ast.Call): + target = ast.unparse(child.func) + if target.rsplit(".", 1)[-1] in names: + actual[(str(path.relative_to(ROOT)), owner, target)] += 1 + assert actual == Counter( + { + ("server/cli/cell_prepare.py", "_prepare_locked", "_write_rows"): 1, + ("server/cli/cell_attach_pg.py", "commit_enforced", "_mode_try"): 1, + ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, + } + ) From 438018601bb9aebe515f7eb3ba280ff4dbfff69a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:24:41 +0000 Subject: [PATCH 041/181] Build placement indexes outside migration transactions Concurrent index builds release the ordinary migration key so a shipped blocking migrator can finish without retaining a snapshot behind the build. Retry from the catalogs and checksum ledger, and acknowledge a WAL fence on recorded reruns. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 29 ++ docs/development/migrations.md | 54 ++- .../004_placements_by_cell_org.sql | 3 + server/core/schema/pg_migrate.py | 154 ++++++-- server/core/schema/pg_online_index.py | 125 ++++++ .../fixtures/pg_migrate_n_minus_one.py | 151 +++++++ tests/integration/test_pg_online_index.py | 373 ++++++++++++++++++ .../test_pg_online_index_durability.py | 203 ++++++++++ tests/unit/test_cell_org_triggers.py | 8 + tests/unit/test_pg_online_index.py | 212 ++++++++++ 10 files changed, 1275 insertions(+), 37 deletions(-) create mode 100644 schemas/router_migrations/004_placements_by_cell_org.sql create mode 100644 server/core/schema/pg_online_index.py create mode 100644 tests/integration/fixtures/pg_migrate_n_minus_one.py create mode 100644 tests/integration/test_pg_online_index.py create mode 100644 tests/integration/test_pg_online_index_durability.py create mode 100644 tests/unit/test_pg_online_index.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22342c2cab..a132706be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -896,10 +896,39 @@ jobs: tests/integration/test_pg_clickhouse_live_ch.py tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py + tests/integration/test_pg_online_index.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_org_triggers_pg.py + - name: pytest online-index durability on a private physical standby + timeout-minutes: 8 + env: + LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} + LOOM_PG_CONTAINER: loom-scale-index-drill-pg + LOOM_PG_HOST_PORT: "55432" + LOOM_PG_INDEX_DRILL: "1" + LOOM_PG_INDEX_DRILL_DSN: postgres://loom:loomdev@localhost:55432/loom + run: | + set -euo pipefail + for name in loom-scale-index-drill-pg loom-scale-index-drill-pg-standby; do + if docker container inspect "$name" >/dev/null 2>&1; then + echo "drill container already exists: $name" >&2 + exit 1 + fi + done + if docker network inspect loom-scale-index-drill-pg-net >/dev/null 2>&1; then + echo "drill network already exists" >&2 + exit 1 + fi + cleanup() { + docker rm -f -v loom-scale-index-drill-pg loom-scale-index-drill-pg-standby || true + docker network rm loom-scale-index-drill-pg-net || true + } + trap cleanup EXIT + bash dev/ci_start_postgres.sh + docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby + pytest -q tests/integration/test_pg_online_index_durability.py # The tenant-isolation gates again, against a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. # They run whether or not the memory suites passed (`!cancelled()`). diff --git a/docs/development/migrations.md b/docs/development/migrations.md index bbea754e87..0fcc560a45 100644 --- a/docs/development/migrations.md +++ b/docs/development/migrations.md @@ -157,20 +157,56 @@ operator step ([release process](./release-process.md#release-notes)). edited once merged, and every `CREATE`, `ALTER` or `DROP` of a table names `loom.
`, except a rename's new name, where Postgres takes no schema (`tests/unit/test_storage_layout_rows.py`). A boot applies them under a Postgres -advisory lock, each file in one transaction with its ledger row. A migration waits at +advisory lock, each ordinary file in one transaction with its ledger row. A migration waits at most 5 s for a table lock (`lock_timeout`, then retried), so an `ALTER` never queues every query of the table behind a long transaction. A build serves a ledger that holds a later build's files, with a warning naming them; a ledger file it does not carry numbered at or below its own last file, or one whose checksum changed, refuses the boot. -Every file runs in one transaction, so a migration builds no index on a large table: the -build would hold off the table's writes, every commit answering 503, until it ends. Such -an index needs `CREATE INDEX CONCURRENTLY` outside a transaction, and a boot waiting on -the advisory lock deadlocks with it (a waiting session holds a snapshot the build waits -out): a change that adds such a file also takes the lock by retrying -`pg_try_advisory_lock`. When the first release that writes a table carries the file, the -table is small wherever the file runs, and its index may be built in the file's -transaction (`010_revision_delivered_at.sql`). +Ordinary files run in one transaction, so they build no index on a table that +already ships: a transactional build blocks writes until it ends. An index on a +table introduced by the same release may be transactional because that table is +small wherever the file runs (`010_revision_delivered_at.sql`). + +For an existing table, the shared Postgres runner accepts only this online file +form, in either the cell or router journal: + +```sql +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +``` + +The declaration and statement must name the same schema, table, index and ordered +columns. Only ordinary identifiers, a non-unique btree and C collations are +accepted. Expressions, predicates, included columns and additional statements +are refused before the ledger changes. This is not a general escape from +transactional migrations. + +The runner takes session advisory locks by retrying `pg_try_advisory_lock`, with +no transaction or snapshot between attempts. It releases the ordinary migration +key before acquiring a separate schema-scoped online-index key and running +concurrent DDL. A shipped blocking migrator can therefore finish its unchanged +journal while the index waits for old snapshots. After the build finishes, the +runner reacquires the ordinary key, records the file's checksum in the existing +ledger and commits a WAL-message fence. Later files wait for this ledger entry. + +Catalog validation checks the table, ordered columns and collations, btree +method, absence of constraints and expressions, and valid/ready state. A +matching incomplete index is dropped concurrently and rebuilt; a different +same-named object refuses without being dropped. A recorded but absent or +invalid index refuses as corruption. Even when every file is recorded, a build +carrying an online file validates its index and acknowledges a fresh WAL fence: +a primary-visible ledger row after a cancelled synchronous COMMIT is insufficient. +An interrupted build, drop or ledger write is retried from catalogs and the +existing ledger, without a separate job table. Cancellation terminates the +migration session before asyncpg can wait indefinitely on rollback. + +The router's dedicated migration connection has no statement timeout; its +serving pool retains its statement limit. The runner preserves a borrowed +connection's statement budget, including a stopped restore command's deadline. +Concurrent DDL retains the five-second lock timeout and can be interrupted and +retried without changing the file checksum. ## Schema dump and check diff --git a/schemas/router_migrations/004_placements_by_cell_org.sql b/schemas/router_migrations/004_placements_by_cell_org.sql new file mode 100644 index 0000000000..774efeb883 --- /dev/null +++ b/schemas/router_migrations/004_placements_by_cell_org.sql @@ -0,0 +1,3 @@ +-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 7876c9f178..ebd230036b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -1,6 +1,7 @@ """Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in -order at boot under a session advisory lock, each file in one transaction with -its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +order at boot under a session advisory lock, each ordinary file in a transaction +with its ledger row (`loom.pg_migrations`, the file's SHA-256). Declared online indexes +run concurrently outside transactions under a separate session key. The router's directory (`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks its session as one that ran the tenancy check and, after the files and before the lock @@ -8,13 +9,14 @@ The rules are the ClickHouse journal's (docs/development/migrations.md): additive only, never edit a merged file. A ledger row whose checksum no longer matches its -file refuses the boot, as does a ledger row for a file this build does not carry -when that file's number is above the build's last one (a newer build's schema is -served, as ClickHouse's is; an unknown lower number means the journal was edited). +file refuses the boot, as does an unknown file numbered at or below this build's +last migration. A build serves a newer schema, warning about unknown files numbered +above its last migration. """ from __future__ import annotations +import asyncio import hashlib import logging from asyncio import sleep @@ -24,7 +26,9 @@ from typing import Any from server.core import pg +from server.core.ch import set_query_tag from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema import pg_online_index from server.core.schema.journal import migration_number, sorted_migrations log = logging.getLogger(__name__) @@ -105,32 +109,126 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files = migration_files(mig_dir) - done: list[str] = [] - # Closing the connection ends the session, and with it the advisory lock. + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") async with (connect or pg.dedicated)() as conn: - # Another boot migrating holds the lock for as long as its files take. - await conn.execute("SELECT pg_advisory_lock($1)", lock_key) - if schema == "loom": - # A migration's data steps only copy or rewrite rows already stored here: the - # org triggers pass them (server/core/cell_triggers.py). - await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") - await conn.execute("SET lock_timeout = '5s'") - await conn.execute(_ledger(schema)) - rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") - for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): - await _apply(conn, name, sql, digest, schema=schema) - log.info("applied Postgres migration %s", name) - done.append(name) - if schema == "loom": - # On every migrating boot, after the pending files and before the lock goes. - from server.core import cell_triggers - - await cell_triggers.ensure(conn) - return done + task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) + try: + return await asyncio.shield(task) + except BaseException: + # Terminate before cancellation can enter asyncpg's unbounded rollback. + conn.terminate() + task.cancel() + await asyncio.wait({task}, timeout=1) + if task.done() and not task.cancelled(): + task.exception() + raise + + +async def _lock(conn: Any, key: int) -> None: + while True: + set_query_tag("migrate.postgres.lock") + if await conn.fetchval("SELECT pg_try_advisory_lock($1)", key): + return + # Each try ends its statement/snapshot before another attempt. + await sleep(0.1) + + +async def _unlock(conn: Any, key: int) -> None: + if not await conn.fetchval("SELECT pg_advisory_unlock($1)", key): + raise RuntimeError("Postgres migration advisory lock was lost") -def _read(files: list[Path]) -> list[tuple[str, str, str]]: - return [(f.name, f.read_text(), checksum(f)) for f in files] +def _online_key(schema: str, lock_key: int) -> int: + value = hashlib.sha256(f"loom.online-index:{schema}:{lock_key}".encode()).digest() + key = int.from_bytes(value[:8], "big") & (2**63 - 1) + if key == lock_key: + raise ValueError("Postgres migration lock keys collide") + return key + + +async def _applied(conn: Any, schema: str, files: list[Path]) -> dict[str, str]: + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + applied = {r["filename"]: r["checksum"] for r in rows} + plan(applied, files) + return applied + + +async def _record_online(conn: Any, path: Path, schema: str, *, pending: bool) -> None: + async with conn.transaction(): + if pending: + await conn.execute( + f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)", + path.name, + checksum(path), + ) + # A visible ledger from a cancelled synchronous commit needs a fresh flush. + await conn.execute(pg._DURABLE_MESSAGE) + + +async def _online( + conn: Any, + path: Path, + index: pg_online_index.OnlineIndex, + schema: str, + lock_key: int, + files: list[Path], +) -> bool: + await _unlock(conn, lock_key) + online_key = _online_key(schema, lock_key) + await _lock(conn, online_key) + applied = await _applied(conn, schema, files) + await pg_online_index.build(conn, index, recorded=path.name in applied) + # A shipped blocking migrator may wait here; the concurrent build is over. + await _lock(conn, lock_key) + applied = await _applied(conn, schema, files) + pending = path.name not in applied + await _record_online(conn, path, schema, pending=pending) + await _unlock(conn, online_key) + return pending + + +async def _migrate( + conn: Any, + files: list[Path], + indexes: dict[str, pg_online_index.OnlineIndex | None], + schema: str, + lock_key: int, +) -> list[str]: + done: list[str] = [] + await _lock(conn, lock_key) + if schema == "loom": + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute("SET synchronous_commit = 'on'") + await conn.execute(_ledger(schema)) + applied = await _applied(conn, schema, files) + for path in files: + set_query_tag("migrate.postgres.apply") + index = indexes[path.name] + if index is not None: + changed = await _online(conn, path, index, schema, lock_key, files) + applied = await _applied(conn, schema, files) + else: + changed = path.name not in applied + if changed: + await _apply(conn, path.name, path.read_text(), checksum(path), schema=schema) + if changed: + log.info("applied Postgres migration %s", path.name) + done.append(path.name) + for index in indexes.values(): + set_query_tag("migrate.postgres.verify") + if index is not None and await pg_online_index.ready(conn, index) is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if schema == "loom": + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + # A caller may lend its marked session and retain unrelated session keys. + await _unlock(conn, lock_key) + return done async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: diff --git a/server/core/schema/pg_online_index.py b/server/core/schema/pg_online_index.py new file mode 100644 index 0000000000..47c2b025a5 --- /dev/null +++ b/server/core/schema/pg_online_index.py @@ -0,0 +1,125 @@ +"""The one nontransactional Postgres migration form: a declared concurrent index.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +_ID = r"[a-z_][a-z0-9_]{0,62}" +_DECLARATION = re.compile( + rf"-- loom:online-index (?P{_ID})\.(?P{_ID}) ON " + rf"(?P{_ID})\.(?P
{_ID}) \((?P{_ID}(?:, {_ID})*)\)\n" +) +_CREATE = re.compile( + rf"CREATE\s+INDEX\s+CONCURRENTLY\s+(?P{_ID})\s+ON\s+" + rf"(?P{_ID})\.(?P
{_ID})\s+USING\s+btree\s*" + rf'\((?P{_ID}\s+COLLATE\s+"C"(?:\s*,\s*{_ID}\s+COLLATE\s+"C")*)\)\s*;\s*' +) + + +@dataclass(frozen=True) +class OnlineIndex: + schema: str + name: str + table: str + columns: tuple[str, ...] + sql: str + + @property + def qualified(self) -> str: + return f'"{self.schema}"."{self.name}"' + + +def parse(sql: str) -> OnlineIndex | None: + """Refuse malformed directives before a connection or ledger write is made.""" + if "loom:online-index" not in sql: + return None + declaration = _DECLARATION.match(sql) + create = _CREATE.fullmatch(sql[declaration.end() :]) if declaration else None + if declaration is None or create is None: + raise ValueError("invalid online-index migration declaration or SQL") + columns = tuple(declaration["columns"].split(", ")) + actual = tuple(part.split()[0] for part in create["columns"].split(",")) + if ( + declaration["schema"] != declaration["table_schema"] + or any(declaration[key] != create[key] for key in ("schema", "name", "table")) + or columns != actual + or len(set(columns)) != len(columns) + ): + raise ValueError("online-index SQL does not match its declaration") + return OnlineIndex( + declaration["schema"], + declaration["name"], + declaration["table"], + columns, + sql[declaration.end() :], + ) + + +_CATALOG = """ +SELECT c.relkind::text AS relkind, tn.nspname AS table_schema, t.relname AS table_name, + am.amname, i.indisvalid, i.indisready, i.indislive, + NOT i.indisunique AND NOT i.indisexclusion AND i.indpred IS NULL + AND i.indexprs IS NULL AND i.indnkeyatts = i.indnatts + AND NOT i.indisreplident AND c.reloptions IS NULL AS plain, + ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.n) AS columns, + ARRAY(SELECT co.collname::text FROM unnest(i.indcollation) WITH ORDINALITY k(oid, n) + JOIN pg_collation co ON co.oid = k.oid + JOIN pg_namespace ns ON ns.oid = co.collnamespace AND ns.nspname = 'pg_catalog' + ORDER BY k.n) AS collations, + ARRAY(SELECT v FROM unnest(i.indoption) v) AS options, + ARRAY(SELECT op.opcdefault FROM unnest(i.indclass) WITH ORDINALITY k(oid, n) + JOIN pg_opclass op ON op.oid = k.oid ORDER BY k.n) AS default_opclasses, + EXISTS(SELECT 1 FROM pg_constraint WHERE conindid = c.oid) + OR EXISTS(SELECT 1 FROM pg_depend WHERE classid = 'pg_class'::regclass + AND objid = c.oid AND deptype IN ('e', 'x')) + OR EXISTS(SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) AS dependent +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_index i ON i.indexrelid = c.oid +LEFT JOIN pg_class t ON t.oid = i.indrelid +LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace +LEFT JOIN pg_am am ON am.oid = c.relam +WHERE n.nspname = $1 AND c.relname = $2 +""" + + +async def ready(conn: Any, index: OnlineIndex) -> bool | None: + """Absent, matching incomplete, or matching ready; foreign objects always refuse.""" + row = await conn.fetchrow(_CATALOG, index.schema, index.name) + if row is None: + return None + expected = { + "relkind": "i", + "table_schema": index.schema, + "table_name": index.table, + "amname": "btree", + "plain": True, + "columns": list(index.columns), + "collations": ["C"] * len(index.columns), + "options": [0] * len(index.columns), + "default_opclasses": [True] * len(index.columns), + "dependent": False, + } + differences = {key: row[key] for key, value in expected.items() if row[key] != value} + if differences: + raise ValueError( + f"online-index {index.qualified} has a different definition or dependency: " + f"{differences}" + ) + return bool(row["indisvalid"] and row["indisready"] and row["indislive"]) + + +async def build(conn: Any, index: OnlineIndex, *, recorded: bool) -> None: + state = await ready(conn, index) + if recorded and state is not True: + raise ValueError(f"recorded online-index {index.qualified} is absent or invalid") + if state is True: + return + if state is False: + await conn.execute(f"DROP INDEX CONCURRENTLY {index.qualified}") + await conn.execute(index.sql) + if await ready(conn, index) is not True: + raise ValueError(f"online-index {index.qualified} did not become valid and ready") diff --git a/tests/integration/fixtures/pg_migrate_n_minus_one.py b/tests/integration/fixtures/pg_migrate_n_minus_one.py new file mode 100644 index 0000000000..7876c9f178 --- /dev/null +++ b/tests/integration/fixtures/pg_migrate_n_minus_one.py @@ -0,0 +1,151 @@ +"""Postgres migrations: numbered SQL files in `schemas/pg_migrations/`, applied in +order at boot under a session advisory lock, each file in one transaction with +its ledger row (`loom.pg_migrations`, the file's SHA-256). The router's directory +(`schemas/router_migrations/`, `server/router/directory.py`) migrates the same way, +on its own database, ledger schema and lock. For schema `loom`, `migrate` also marks +its session as one that ran the tenancy check and, after the files and before the lock +goes, runs `cell_triggers.ensure` (docs/architecture/cells.md). + +The rules are the ClickHouse journal's (docs/development/migrations.md): additive +only, never edit a merged file. A ledger row whose checksum no longer matches its +file refuses the boot, as does a ledger row for a file this build does not carry +when that file's number is above the build's last one (a newer build's schema is +served, as ClickHouse's is; an unknown lower number means the journal was edited). +""" + +from __future__ import annotations + +import hashlib +import logging +from asyncio import sleep +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Any + +from server.core import pg +from server.core.paths import PG_MIGRATIONS_DIR +from server.core.schema.journal import migration_number, sorted_migrations + +log = logging.getLogger(__name__) + +# A fixed key for pg_advisory_lock: every Loom process migrates under it. +_LOCK_KEY = 0x6C6F6F6D_6D696772 # "loommigr" + + +def _ledger(schema: str) -> str: + # `schema` is one of this build's constants (`loom`, `loom_router`), never input. + return f""" +CREATE SCHEMA IF NOT EXISTS {schema}; +CREATE TABLE IF NOT EXISTS {schema}.pg_migrations ( + filename text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)""" + + +class MigrationDrift(RuntimeError): + """An applied file changed, or the ledger holds a file the journal lost.""" + + +def migration_files(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[Path]: + return sorted_migrations(mig_dir.glob("*.sql")) + + +def checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def plan(applied: dict[str, str], files: list[Path]) -> list[Path]: + """The files still to apply; raises MigrationDrift on a changed or lost file.""" + known = {f.name: f for f in files} + last = max((migration_number(f.name) or 0 for f in files), default=0) + for name, digest in applied.items(): + f = known.get(name) + if f is None: + if (migration_number(name) or 0) <= last: + raise MigrationDrift(f"applied Postgres migration {name} is not in this build") + continue + if checksum(f) != digest: + raise MigrationDrift(f"applied Postgres migration {name} was edited") + newer = sorted(n for n in applied if n not in known) + if newer: + log.warning("Postgres schema has migrations this build does not carry: %s", newer) + return [f for f in files if f.name not in applied] + + +async def pending(mig_dir: Path = PG_MIGRATIONS_DIR) -> list[str]: + """The files `migrate` would apply, read-only: every file when the ledger is not + there yet. For a process that applies none (`LOOM_BOOT_MIGRATE=0`).""" + import asyncpg + + try: + rows = await pg.fetch("SELECT filename, checksum FROM loom.pg_migrations") + except asyncpg.exceptions.UndefinedTableError: + rows = [] + applied = {r["filename"]: r["checksum"] for r in rows} + return [f.name for f in plan(applied, migration_files(mig_dir))] + + +async def migrate( + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + connect: Callable[[], AbstractAsyncContextManager[Any]] | None = None, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply every pending file; returns their names. Concurrent boots serialize on + the advisory lock and the later ones find nothing pending. `connect`, `schema` + and `lock_key` name another database's connection, ledger and lock (the router's + directory); the defaults are the memory authority's. + + On a connection of its own, with no client command timeout and no statement + timeout: DDL on a large table runs as long as it takes. A lock it waits for is + bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never + holds every read and write of the table behind it; the file is retried. For schema + `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" + files = migration_files(mig_dir) + done: list[str] = [] + # Closing the connection ends the session, and with it the advisory lock. + async with (connect or pg.dedicated)() as conn: + # Another boot migrating holds the lock for as long as its files take. + await conn.execute("SELECT pg_advisory_lock($1)", lock_key) + if schema == "loom": + # A migration's data steps only copy or rewrite rows already stored here: the + # org triggers pass them (server/core/cell_triggers.py). + await conn.execute(f"SET {pg.TENANCY_MARKER} = 'on'") + await conn.execute("SET lock_timeout = '5s'") + await conn.execute(_ledger(schema)) + rows = await conn.fetch(f"SELECT filename, checksum FROM {schema}.pg_migrations") + for name, sql, digest in _read(plan({r["filename"]: r["checksum"] for r in rows}, files)): + await _apply(conn, name, sql, digest, schema=schema) + log.info("applied Postgres migration %s", name) + done.append(name) + if schema == "loom": + # On every migrating boot, after the pending files and before the lock goes. + from server.core import cell_triggers + + await cell_triggers.ensure(conn) + return done + + +def _read(files: list[Path]) -> list[tuple[str, str, str]]: + return [(f.name, f.read_text(), checksum(f)) for f in files] + + +async def _apply(conn: Any, name: str, sql: str, digest: str, *, schema: str = "loom") -> None: + import asyncpg + + record = f"INSERT INTO {schema}.pg_migrations (filename, checksum) VALUES ($1, $2)" + attempts = 5 # tries before the boot fails on a table lock held past `lock_timeout` + for attempt in range(1, attempts + 1): + try: + async with conn.transaction(): + await conn.execute(sql) + await conn.execute(record, name, digest) + return + except asyncpg.exceptions.LockNotAvailableError: + if attempt == attempts: + raise + log.warning("Postgres migration %s waited too long for a lock; retrying", name) + await sleep(attempt) diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py new file mode 100644 index 0000000000..af93dd0937 --- /dev/null +++ b/tests/integration/test_pg_online_index.py @@ -0,0 +1,373 @@ +"""Online index retries and migrator coexistence on scratch Postgres databases.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate, pg_online_index +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + conn = await asyncpg.connect(dsn, server_settings={"application_name": "online-index-test"}) + try: + yield conn + finally: + await conn.close() + + try: + async with connect() as conn: + yield conn, connect + finally: + await directory.close() + await drop_database(admin, name) + + +def _prefix(tmp_path: Path) -> Path: + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + return prefix + + +async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: + return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) + + +async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None and await pg_online_index.ready(conn, index) is True + assert await conn.fetchval("SELECT to_regclass('loom_router.placements_by_cell') IS NOT NULL") + + +async def test_install_from_empty(database) -> None: + _, connect = database + assert len(await _migrate(connect)) == 4 + + +async def _until(conn: Any, sql: str, *args: Any) -> None: + async with asyncio.timeout(8): + for _ in range(800): + if await conn.fetchval(sql, *args): + return + await asyncio.sleep(0.01) + pytest.fail("Postgres did not reach the expected migration state") + + +def _old_runner(): + """The shipped blocking runner, kept as source for the rolling-upgrade proof.""" + import importlib.util + + path = Path(__file__).parent / "fixtures" / "pg_migrate_n_minus_one.py" + spec = importlib.util.spec_from_file_location("old_pg_migrate", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database, tmp_path) -> None: + conn, connect = database + old_files = _prefix(tmp_path) + await _migrate(connect, old_files) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + first = asyncio.create_task(_migrate(connect)) + second = None + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + second = asyncio.create_task(_migrate(connect)) + # The ordinary key is free while CREATE INDEX waits for this old snapshot. + # Running this original blocking implementation would deadlock if it weren't. + old = _old_runner() + async with asyncio.timeout(2): + assert ( + await old.migrate( + old_files, connect=connect, schema="loom_router", lock_key=8917423 + ) + == [] + ) + assert not first.done() and not second.done() + # Even while the index waits, the old serving path can write and read. + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, host, state, credential_sha256, deployment_id) " + "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" + ) + assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await directory.lookup("org-a") == ("a", "a.example.test") + await tx.rollback() + async with asyncio.timeout(5): + results = await asyncio.gather(first, second) + assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + finally: + if snapshot.is_in_transaction(): + await tx.rollback() + for task in (first, second): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(t for t in (first, second) if t is not None), return_exceptions=True + ) + + +async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs_it( + database, tmp_path +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + async with connect() as snapshot: + tx = snapshot.transaction(isolation="repeatable_read") + await tx.start() + await snapshot.fetchval("SELECT count(*) FROM loom_router.placements") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_progress_create_index " + "WHERE datid = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND phase = 'waiting for old snapshots')", + ) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 2) + assert await conn.fetchval( + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" + ) + assert not await conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql')" + ) + finally: + await tx.rollback() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == [] + + +@pytest.mark.parametrize( + "definition", + [ + "CREATE TABLE loom_router.placements_by_cell_org (id int)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (org, cell)", + 'CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org COLLATE "POSIX")', + "CREATE UNIQUE INDEX placements_by_cell_org ON loom_router.placements (cell, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WHERE org > 'm'", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) INCLUDE(placed_at)", + "CREATE INDEX placements_by_cell_org ON loom_router.cells (cell, host)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell DESC, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell text_pattern_ops, org)", + "CREATE INDEX placements_by_cell_org ON loom_router.placements (cell, org) WITH (fillfactor=70)", + ], +) +async def test_foreign_same_named_object_is_never_dropped(database, tmp_path, definition) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + await conn.execute(definition) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + with pytest.raises(ValueError, match="different definition"): + await _migrate(connect) + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_recorded_invalid_index_and_checksum_drift_refuse(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute("DROP INDEX loom_router.placements_by_cell_org") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect) + await conn.execute( + "UPDATE loom_router.pg_migrations SET checksum = repeat('0', 64) " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + with pytest.raises(pg_migrate.MigrationDrift): + await _migrate(connect) + + +async def test_cancel_after_catalog_completion_reuses_the_valid_index( + database, tmp_path, monkeypatch +) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + original = pg_online_index.build + + async def completed_then_cancel(*args, **kwargs): + await original(*args, **kwargs) + raise asyncio.CancelledError + + monkeypatch.setattr(pg_online_index, "build", completed_then_cancel) + with pytest.raises(asyncio.CancelledError): + await _migrate(connect) + oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") + monkeypatch.setattr(pg_online_index, "build", original) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid + + +async def test_keyset_plan_uses_the_ordered_cell_index(database) -> None: + conn, connect = database + await _migrate(connect) + await conn.execute( + "INSERT INTO loom_router.cells (cell, host, state, credential_sha256, deployment_id) " + "SELECT 'cell-' || n, 'cell-' || n || '.example.test', 'serving', " + "md5(n::text) || md5(n::text), 'dep-' || n FROM generate_series(1, 32) n" + ) + await conn.execute( + "INSERT INTO loom_router.placements(org, cell) " + "SELECT 'org-' || lpad(n::text, 5, '0'), 'cell-' || (n % 32 + 1) " + "FROM generate_series(1, 4096) n" + ) + await conn.execute("VACUUM ANALYZE loom_router.placements") + plan = await conn.fetch( + "EXPLAIN SELECT org FROM loom_router.placements " + "WHERE cell = 'cell-2' AND org > 'org-01000' ORDER BY org LIMIT 10" + ) + text = "\n".join(r[0] for r in plan) + assert "placements_by_cell_org" in text and "Sort" not in text + + +async def test_ordinary_files_still_rollback_with_the_ledger(database, tmp_path) -> None: + conn, connect = database + (tmp_path / "001_one.sql").write_text("CREATE TABLE loom_router.one (id int);") + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int); SELECT 1 / 0;") + import asyncpg + + with pytest.raises(asyncpg.DivisionByZeroError): + await _migrate(connect, tmp_path) + assert await conn.fetchval("SELECT to_regclass('loom_router.one') IS NOT NULL") + assert not await conn.fetchval("SELECT to_regclass('loom_router.two') IS NOT NULL") + assert await conn.fetchval("SELECT count(*) FROM loom_router.pg_migrations") == 1 + (tmp_path / "002_two.sql").write_text("CREATE TABLE loom_router.two (id int);") + assert await _migrate(connect, tmp_path) == ["002_two.sql"] + + +async def test_invalid_online_declaration_fails_before_the_ledger_exists( + database, tmp_path +) -> None: + conn, connect = database + (tmp_path / "001_bad.sql").write_text("-- loom:online-index bad\nSELECT 1;") + with pytest.raises(ValueError, match="online-index"): + await _migrate(connect, tmp_path) + assert not await conn.fetchval("SELECT to_regclass('loom_router.pg_migrations') IS NOT NULL") + + +async def test_cancelled_concurrent_drop_is_retryable(database, tmp_path) -> None: + conn, connect = database + await _migrate(connect, _prefix(tmp_path)) + index = pg_online_index.parse( + (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() + ) + assert index is not None + await conn.execute(index.sql) + async with connect() as reader: + tx = reader.transaction() + await tx.start() + await reader.execute("SET LOCAL enable_seqscan = off") + plan = await reader.fetch( + "EXPLAIN SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org" + ) + assert "placements_by_cell_org" in "\n".join(r[0] for r in plan) + await reader.fetch("SELECT org FROM loom_router.placements WHERE cell = 'a' ORDER BY org") + + async def direct_drop(): + async with connect() as dropping: + try: + await dropping.execute( + "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" + ) + finally: + dropping.terminate() + + dropping = asyncio.create_task(direct_drop()) + try: + query = ( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND query LIKE 'DROP INDEX CONCURRENTLY%' " + "AND wait_event IS NOT NULL)" + ) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + assert await pg_online_index.ready(conn, index) is False + dropping = asyncio.create_task(_migrate(connect)) + await _until(conn, query) + dropping.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dropping, 2) + finally: + await tx.rollback() + if not dropping.done(): + dropping.cancel() + await asyncio.gather(dropping, return_exceptions=True) + assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + + +async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: + conn, connect = database + await conn.execute("SELECT pg_advisory_lock(8917423::bigint)") + migrating = asyncio.create_task(_migrate(connect)) + try: + await _until( + conn, + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid() " + "AND query = 'SELECT pg_try_advisory_lock($1)' " + "AND state = 'idle' AND backend_xmin IS NULL AND xact_start IS NULL)", + ) + assert not migrating.done() + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + assert len(await asyncio.wait_for(migrating, 5)) == 4 + finally: + await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +async def test_online_validation_is_repeated_after_later_files(database, tmp_path) -> None: + _, connect = database + for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): + (tmp_path / source.name).write_bytes(source.read_bytes()) + (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + with pytest.raises(ValueError, match="recorded online-index"): + await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py new file mode 100644 index 0000000000..e5644af5af --- /dev/null +++ b/tests/integration/test_pg_online_index_durability.py @@ -0,0 +1,203 @@ +"""A cancelled online-index ledger commit survives a private standby promotion. + +Only its dedicated CI pair enables this destructive drill. Configured failures +are errors; the adopted integration server's replication settings never change. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +from collections.abc import AsyncIterator +from pathlib import Path +from urllib.parse import urlsplit + +import asyncpg +import pytest + +from server.core.paths import ROUTER_MIGRATIONS_DIR +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database + +PRIMARY = "loom-scale-index-drill-pg" +STANDBY = PRIMARY + "-standby" +APPLICATION = "online-index-durability" + + +async def _docker(*args: str) -> str: + process = await asyncio.create_subprocess_exec( + "docker", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + try: + async with asyncio.timeout(30): + stdout, stderr = await process.communicate() + except BaseException: + if process.returncode is None: + process.kill() + await asyncio.wait_for(process.wait(), 5) + raise + assert process.returncode == 0, stderr.decode() + return stdout.decode().strip() + + +async def _sql(container: str, database: str, sql: str) -> str: + return await _docker("exec", container, "psql", "-U", "loom", "-d", database, "-Atc", sql) + + +async def _waiting(observer, task) -> int: + async with asyncio.timeout(8): + for _ in range(800): + assert not task.done(), "migration acknowledged before standby flush" + pid = await observer.fetchval( + "SELECT pid FROM pg_stat_activity WHERE application_name = $1 " + "AND wait_event = 'SyncRep'", + APPLICATION, + ) + if pid is not None: + assert not task.done() + return pid + await asyncio.sleep(0.01) + pytest.fail("migration did not reach its synchronous COMMIT") + + +async def _ownership(observer) -> None: + identifier = str(await observer.fetchval("SELECT system_identifier FROM pg_control_system()")) + for container in (PRIMARY, STANDBY): + assert await _docker("inspect", "--format", "{{.Name}}", container) == "/" + container + assert ( + await _sql(container, "loom", "SELECT system_identifier FROM pg_control_system()") + == identifier + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "t" + assert ( + await observer.fetchval( + "SELECT count(*) FROM pg_stat_replication WHERE application_name = 'loom_standby' " + "AND sync_state = 'sync' AND state = 'streaming'" + ) + == 1 + ) + + +@pytest.mark.asyncio +@pytest.mark.recovery +@pytest.mark.destructive +async def test_online_ledger_retry_survives_physical_standby_promotion( + tmp_path, monkeypatch +) -> None: + if not os.environ.get("LOOM_PG_INDEX_DRILL"): + pytest.skip("LOOM_PG_INDEX_DRILL not set: needs its disposable physical standby") + assert os.environ["LOOM_PG_INDEX_DRILL"] == "1" + dsn = os.environ["LOOM_PG_INDEX_DRILL_DSN"] + observer = await asyncpg.connect(dsn, timeout=5, command_timeout=5) + tasks = [] + paused = False + try: + await _ownership(observer) + _, target = await create_database(dsn) + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[asyncpg.Connection]: + conn = await asyncpg.connect( + target, + timeout=5, + server_settings={"application_name": APPLICATION, "synchronous_commit": "on"}, + ) + try: + yield conn + finally: + conn.terminate() + + prefix = tmp_path / "old" + prefix.mkdir() + for source in sorted(ROUTER_MIGRATIONS_DIR.glob("*.sql"))[:3]: + (prefix / source.name).write_bytes(source.read_bytes()) + + async def migrate(files: Path = ROUTER_MIGRATIONS_DIR): + return await pg_migrate.migrate( + files, connect=connect, schema="loom_router", lock_key=8917423 + ) + + await migrate(prefix) + record = pg_migrate._record_online + + async def pause_before_ledger(*args, **kwargs): + nonlocal paused + await _docker("pause", STANDBY) + paused = True + await record(*args, **kwargs) + + monkeypatch.setattr(pg_migrate, "_record_online", pause_before_ledger) + first = asyncio.create_task(migrate()) + tasks.append(first) + pid = await _waiting(observer, first) + # The index is acknowledged already; its ledger COMMIT alone is unflushed. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(first, 3) + async with asyncio.timeout(5): + for _ in range(500): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid = $1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("cancelled migration retained backend locks") + monkeypatch.setattr(pg_migrate, "_record_online", record) + retry = asyncio.create_task(migrate()) + tasks.append(retry) + await _waiting(observer, retry) + await _docker("unpause", STANDBY) + paused = False + assert await asyncio.wait_for(retry, 8) == [] + observer.terminate() + await _docker("kill", "--signal", "KILL", PRIMARY) + await _docker( + "exec", + "--user", + "postgres", + STANDBY, + "bash", + "-c", + 'pg_ctl -D "$PGDATA" promote -w -t 10', + ) + assert await _sql(STANDBY, "loom", "SELECT pg_is_in_recovery()") == "f" + database = urlsplit(target).path.removeprefix("/") + assert ( + await _sql( + STANDBY, + database, + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql' AND checksum = '" + + pg_migrate.checksum(ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql") + + "'", + ) + == "1" + ) + assert ( + await _sql( + STANDBY, + database, + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass", + ) + == "t" + ) + finally: + if paused: + await _docker("unpause", STANDBY) + for task in tasks: + if not task.done(): + task.cancel() + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + observer.terminate() diff --git a/tests/unit/test_cell_org_triggers.py b/tests/unit/test_cell_org_triggers.py index 769cfe1bb3..5d25ba0177 100644 --- a/tests/unit/test_cell_org_triggers.py +++ b/tests/unit/test_cell_org_triggers.py @@ -86,6 +86,14 @@ async def execute(self, sql: str, *_a: Any) -> None: async def fetch(self, _sql: str, *_a: Any) -> list[Any]: return [] + def terminate(self) -> None: + self.statements.append("terminated") + + async def fetchval(self, sql: str, *_a: Any) -> bool: + assert sql in ("SELECT pg_try_advisory_lock($1)", "SELECT pg_advisory_unlock($1)") + self.statements.append(sql) + return True + @pytest.mark.parametrize("name", ["eu-1", ""]) @pytest.mark.parametrize("schema", ["loom", "loom_router"]) diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py new file mode 100644 index 0000000000..7fc43fa2bb --- /dev/null +++ b/tests/unit/test_pg_online_index.py @@ -0,0 +1,212 @@ +"""Only the declared concurrent btree form escapes a migration transaction.""" + +from __future__ import annotations + +import pytest + +from server.core.schema import pg_online_index + +SQL = """-- loom:online-index loom_router.placements_by_cell_org ON loom_router.placements (cell, org) +CREATE INDEX CONCURRENTLY placements_by_cell_org ON loom_router.placements +USING btree (cell COLLATE "C", org COLLATE "C"); +""" + + +def test_the_online_index_declaration_has_one_exact_target() -> None: + parsed = pg_online_index.parse(SQL) + assert parsed is not None + assert parsed.schema == "loom_router" + assert parsed.name == "placements_by_cell_org" + assert parsed.table == "placements" + assert parsed.columns == ("cell", "org") + assert pg_online_index.parse("CREATE TABLE loom.t (id int);") is None + + +@pytest.mark.parametrize( + "sql", + [ + SQL + "SELECT 1;", + SQL.replace("CREATE INDEX", "CREATE UNIQUE INDEX"), + SQL.replace('org COLLATE "C"', 'org COLLATE "POSIX"'), + SQL.replace("(cell, org)", "(org, cell)"), + SQL.replace("ON loom_router.placements\n", "ON loom_router.other\n"), + SQL.replace("USING btree", "USING hash"), + SQL.replace('org COLLATE "C"', 'lower(org) COLLATE "C"'), + SQL.replace(");", ") WHERE true;"), + SQL.replace("-- loom:online-index", "-- loom:online-index-bad"), + SQL.replace("placements_by_cell_org ON", "other_index ON", 1), + SQL.replace("(cell, org)", "(cell, cell)"), + SQL.replace("CONCURRENTLY", ""), + ], +) +def test_other_online_sql_is_refused(sql: str) -> None: + with pytest.raises(ValueError, match="online.index"): + pg_online_index.parse(sql) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("pending", [True, False]) +async def test_ledger_retries_commit_a_wal_fence_even_when_already_recorded( + tmp_path, pending +) -> None: + import contextlib + + from server.core import pg + from server.core.schema import pg_migrate + + path = tmp_path / "001_index.sql" + path.write_text(SQL) + + class Connection: + def __init__(self): + self.statements = [] + + def transaction(self): + return contextlib.nullcontext() + + async def execute(self, sql, *args): + self.statements.append((sql, args)) + + conn = Connection() + await pg_migrate._record_online(conn, path, "loom_router", pending=pending) + assert conn.statements[-1] == (pg._DURABLE_MESSAGE, ()) + assert len(conn.statements) == (2 if pending else 1) + if pending: + assert conn.statements[0][1] == (path.name, pg_migrate.checksum(path)) + + +@pytest.mark.asyncio +async def test_migrator_preserves_a_borrowed_sessions_statement_budget(tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Connection: + def __init__(self): + self.statements = [] + + async def execute(self, sql, *args): + self.statements.append(sql) + + async def fetch(self, sql): + return [] + + async def fetchval(self, sql, *args): + self.statements.append(sql) + return True + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + assert await pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") == [] + assert not any("statement_timeout" in sql for sql in conn.statements) + assert "SET synchronous_commit = 'on'" in conn.statements + assert conn.statements[-1] == "SELECT pg_advisory_unlock($1)" + + +@pytest.mark.asyncio +async def test_online_target_must_match_its_ledger_before_connecting(tmp_path) -> None: + from server.core.schema import pg_migrate + + (tmp_path / "001_index.sql").write_text(SQL) + + def never_connect(): + pytest.fail("invalid target opened a connection") + + with pytest.raises(ValueError, match="ledger schema"): + await pg_migrate.migrate(tmp_path, schema="loom", connect=never_connect) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) +async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( + tmp_path, phase +) -> None: + import asyncio + import contextlib + from types import SimpleNamespace + + from asyncpg.transaction import Transaction + + from server.core.schema import pg_migrate + + (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") + reached = asyncio.Event() + terminated = asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def is_closed(self): + return terminated.is_set() + + def transaction(self): + return Transaction(self, None, False, False) + + def terminate(self): + terminated.set() + + async def fetchval(self, sql, *args): + return True + + async def fetch(self, sql): + return [] + + async def execute(self, sql, *args): + if sql == "SELECT stalled_statement;" and phase == "rollback": + raise ValueError("statement failed before rollback") + if ( + (sql == "SELECT stalled_statement;" and phase == "statement") + or (sql.startswith("ROLLBACK") and phase == "rollback") + or (sql.startswith("COMMIT") and phase == "commit") + ): + reached.set() + # Transport termination, rather than a second task cancellation, + # releases a transaction that has already begun async cleanup. + while not terminated.is_set(): + try: + await terminated.wait() + except asyncio.CancelledError: + continue + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + migrating = asyncio.create_task( + pg_migrate.migrate(tmp_path, connect=connect, schema="loom_router") + ) + try: + await asyncio.wait_for(reached.wait(), 1) + migrating.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(migrating, 1) + assert terminated.is_set() + finally: + terminated.set() + if not migrating.done(): + migrating.cancel() + await asyncio.gather(migrating, return_exceptions=True) + + +def test_postgres_journals_keep_distinct_numbers_and_declared_online_forms() -> None: + from server.core.paths import PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR + from server.core.schema import pg_migrate + from server.core.schema.journal import migration_number + + for directory in (PG_MIGRATIONS_DIR, ROUTER_MIGRATIONS_DIR): + files = pg_migrate.migration_files(directory) + numbers = [migration_number(path.name) for path in files] + assert None not in numbers and len(numbers) == len(set(numbers)) + for path in files: + sql = path.read_text() + parsed = pg_online_index.parse(sql) + if "CONCURRENTLY" in sql: + assert parsed is not None, f"{path.name} lacks an online-index declaration" From 417712d4ecde8b788ca060c853ec02e528e42487 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:40:22 +0000 Subject: [PATCH 042/181] Close owned Postgres sessions before cancelling their work Send public driver cancellation before aborting the transport, within one absolute cleanup deadline. Separate borrowed migration execution so its operator keeps sole session supervision. Verify backend lock release and preserve interrupted migration errors. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/core/pg.py | 32 ++++ server/core/schema/pg_migrate.py | 52 +++++-- tests/integration/test_pg_online_index.py | 10 +- tests/integration/test_pg_owned_cleanup.py | 173 +++++++++++++++++++++ tests/unit/test_pg_close_owned.py | 144 +++++++++++++++++ tests/unit/test_pg_online_index.py | 21 ++- 7 files changed, 413 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_pg_owned_cleanup.py create mode 100644 tests/unit/test_pg_close_owned.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a132706be6..6e866c99a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,6 +897,7 @@ jobs: tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py tests/integration/test_pg_online_index.py + tests/integration/test_pg_owned_cleanup.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py diff --git a/server/core/pg.py b/server/core/pg.py index 576b1b1e4b..5daeb33a55 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -21,6 +21,7 @@ import asyncio import json import logging +import time import weakref from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager, suppress @@ -379,6 +380,37 @@ def _terminate(conn: Any) -> None: conn.terminate() +def _retrieve_task(task: asyncio.Task[Any]) -> None: + if not task.cancelled(): + task.exception() + + +async def close_owned(conn: Any, *, end: float) -> None: + """Close an owned session by an absolute time.monotonic deadline. + + Public close sends cancellation before transport cleanup can discard it. Abort + is a bounded fallback, not evidence that the backend has released its locks. + The owner preserves an earlier operation failure when this cleanup fails. + """ + task: asyncio.Task[Any] | None = None + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task = asyncio.create_task(conn.close(timeout=remaining)) + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task.result() + except BaseException: + _terminate(conn) + if task is not None: + if not task.done(): + task.cancel() + task.add_done_callback(_retrieve_task) + raise + + async def _release(pool: Any, conn: Any) -> None: """Back to the pool within a bound: a statement still in flight (a client timeout, a cancelled request) has its cancel sent and acknowledged and the connection reset; diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index ebd230036b..371f61a449 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -19,6 +19,7 @@ import asyncio import hashlib import logging +import time from asyncio import sleep from collections.abc import Callable from contextlib import AbstractAsyncContextManager @@ -108,25 +109,54 @@ async def migrate( bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" - files = migration_files(mig_dir) - # Validate every online declaration before even creating the ledger. - indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} - if any(index is not None and index.schema != schema for index in indexes.values()): - raise ValueError("online-index target differs from its ledger schema") + files, indexes = _validated(mig_dir, schema) async with (connect or pg.dedicated)() as conn: task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) try: return await asyncio.shield(task) except BaseException: - # Terminate before cancellation can enter asyncpg's unbounded rollback. - conn.terminate() - task.cancel() - await asyncio.wait({task}, timeout=1) - if task.done() and not task.cancelled(): - task.exception() + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. + pg._terminate(conn) raise +def _validated( + mig_dir: Path, schema: str +) -> tuple[list[Path], dict[str, pg_online_index.OnlineIndex | None]]: + files = migration_files(mig_dir) + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") + return files, indexes + + +async def migrate_borrowed( + conn: Any, + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply on the owner's session; its supervisor owns all failure cleanup. + + The caller retains its operation deadline and must close the session before + cancelling this work. Normal completion releases only the migration key. + """ + files, indexes = _validated(mig_dir, schema) + return await _migrate(conn, files, indexes, schema, lock_key) + + async def _lock(conn: Any, key: int) -> None: while True: set_query_tag("migrate.postgres.lock") diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index af93dd0937..1995e73bee 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -5,12 +5,14 @@ import asyncio import contextlib import os +import time from collections.abc import AsyncIterator from pathlib import Path from typing import Any import pytest +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.core.schema import pg_migrate, pg_online_index from server.router import directory @@ -315,7 +317,7 @@ async def direct_drop(): "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" ) finally: - dropping.terminate() + await pg.close_owned(dropping, end=time.monotonic() + 1) dropping = asyncio.create_task(direct_drop()) try: @@ -325,6 +327,12 @@ async def direct_drop(): "AND wait_event IS NOT NULL)" ) await _until(conn, query) + # Cancel after catalog invalidation, not an earlier lock/I/O wait. + await _until( + conn, + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid='loom_router.placements_by_cell_org'::regclass", + ) dropping.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(dropping, 2) diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py new file mode 100644 index 0000000000..9085fce011 --- /dev/null +++ b/tests/integration/test_pg_owned_cleanup.py @@ -0,0 +1,173 @@ +"""Cancellation of owned Postgres work releases its actual backend locks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database() -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + observer = await asyncpg.connect(dsn) + try: + await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + yield conn, observer + finally: + await conn.close(timeout=1) + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def sleeping(observer: Any, pid: int) -> None: + async with asyncio.timeout(2): + for _ in range(400): + if await observer.fetchval( + "SELECT wait_event='PgSleep' FROM pg_stat_activity WHERE pid=$1", pid + ): + break + await asyncio.sleep(0.005) + else: + pytest.fail("backend did not reach pg_sleep") + assert await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + + +async def released(observer: Any, pid: int) -> None: + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +async def test_owned_migrator_cancellation_releases_server_locks(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await sleeping(observer, conn.get_server_pid()) + pid = conn.get_server_pid() + started = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - started < 1.5 + await released(observer, pid) + + +@pytest.mark.parametrize("transaction_context", [False, True]) +async def test_public_owned_close_releases_live_query_before_task_cancel( + database, transaction_context: bool +) -> None: + conn, observer = database + pid = conn.get_server_pid() + + async def work() -> None: + if transaction_context: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + else: + await conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_leaves_failure_cleanup_to_owner(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + task = asyncio.create_task(pg_migrate.migrate_borrowed(conn, tmp_path, schema="test")) + pid = conn.get_server_pid() + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_normal_completion_retains_owners_lock(database, tmp_path) -> None: + conn, observer = database + await conn.execute("SELECT pg_advisory_lock(712398)") + (tmp_path / "001_write.sql").write_text("INSERT INTO probe VALUES(2)") + assert await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") == ["001_write.sql"] + assert not conn.is_closed() + assert await observer.fetchval("SELECT count(*) FROM probe") == 2 + assert await conn.fetchval("SELECT pg_advisory_unlock(712398)") + assert ( + await conn.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=pg_backend_pid() AND locktype='advisory'" + ) + == 0 + ) + + +async def test_owned_close_interrupts_installed_transaction_rollback(database, monkeypatch) -> None: + import asyncpg + + conn, observer = database + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + + async def stalled_rollback(self, query, *args, **kwargs): + if self is conn and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + + async def work() -> None: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise ValueError("statement failure before rollback") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_failure_keeps_the_session_for_its_owner(database, tmp_path) -> None: + import asyncpg + + conn, _ = database + (tmp_path / "001_failure.sql").write_text("SELECT * FROM missing_table_for_failure") + with pytest.raises(asyncpg.UndefinedTableError): + await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") + assert not conn.is_closed() + assert await conn.fetchval("SELECT pg_advisory_unlock($1)", pg_migrate._LOCK_KEY) + assert await conn.fetchval("SELECT count(*) FROM test.pg_migrations") == 0 diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py new file mode 100644 index 0000000000..fbbe778a5d --- /dev/null +++ b/tests/unit/test_pg_close_owned.py @@ -0,0 +1,144 @@ +"""Owned-session cleanup delivers cancellation before aborting its transport.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import pytest + +from server.core import pg + +pytestmark = pytest.mark.asyncio + + +class Connection: + def __init__(self) -> None: + self.started = asyncio.Event() + self.finished = asyncio.Event() + self.aborted = False + self.timeouts: list[float] = [] + + async def close(self, *, timeout: float) -> None: # noqa: ASYNC109 — driver API. + self.timeouts.append(timeout) + self.started.set() + await self.finished.wait() + + def terminate(self) -> None: + self.aborted = True + self.finished.set() + + +async def test_public_close_completes_before_transport_abort() -> None: + conn = Connection() + conn.finished.set() + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.timeouts and 0 < conn.timeouts[0] <= 1 + assert not conn.aborted + + +async def test_expired_deadline_aborts_without_starting_close() -> None: + conn = Connection() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() - 1) + assert conn.aborted and not conn.timeouts + + +async def test_close_timeout_aborts_within_the_original_deadline() -> None: + conn = Connection() + started = time.monotonic() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=started + 0.03) + assert conn.aborted and time.monotonic() - started < 0.3 + + +async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: + conn = Connection() + supervisor_started = asyncio.Event() + original = asyncio.CancelledError("operation cancelled") + + async def supervisor() -> None: + try: + supervisor_started.set() + await asyncio.Future() + except asyncio.CancelledError: + try: + await pg.close_owned(conn, end=time.monotonic() + 1) + except BaseException: # noqa: BLE001 — preserve the original cancellation. + pass + raise original from None + + task = asyncio.create_task(supervisor()) + await supervisor_started.wait() + task.cancel() + await asyncio.wait_for(conn.started.wait(), 1) + task.cancel() + with pytest.raises(asyncio.CancelledError, match="operation cancelled"): + await asyncio.wait_for(task, 0.3) + assert conn.aborted + + +async def test_failed_public_close_aborts_and_reports_failure() -> None: + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise ValueError("close failed") + + conn = Broken() + with pytest.raises(ValueError, match="close failed"): + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.aborted + + +async def test_expired_close_child_exception_is_retrieved() -> None: + release = asyncio.Event() + errors: list[dict[str, Any]] = [] + + class LateFailure(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + try: + await release.wait() + except asyncio.CancelledError: + await release.wait() + raise ValueError("late close failure") + + conn = LateFailure() + loop = asyncio.get_running_loop() + old_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _, context: errors.append(context)) + try: + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() + 0.01) + assert conn.aborted + release.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not errors + finally: + release.set() + loop.set_exception_handler(old_handler) + + +async def test_migration_failure_survives_cleanup_failure(monkeypatch, tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise RuntimeError("cleanup failed") + + conn = Broken() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + async def fail(*_: Any) -> None: + raise ValueError("migration failed") + + monkeypatch.setattr(pg_migrate, "_migrate", fail) + with pytest.raises(ValueError, match="migration failed"): + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert conn.aborted diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py index 7fc43fa2bb..3bb0c19476 100644 --- a/tests/unit/test_pg_online_index.py +++ b/tests/unit/test_pg_online_index.py @@ -122,7 +122,7 @@ def never_connect(): @pytest.mark.asyncio @pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) -async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( +async def test_cancellation_closes_before_installed_asyncpg_transaction_unwind( tmp_path, phase ) -> None: import asyncio @@ -136,6 +136,7 @@ async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwi (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") reached = asyncio.Event() terminated = asyncio.Event() + closed = asyncio.Event() class Connection: _pool_release_ctr = 0 @@ -143,11 +144,16 @@ class Connection: _protocol = SimpleNamespace(is_in_transaction=lambda: False) def is_closed(self): - return terminated.is_set() + return closed.is_set() or terminated.is_set() def transaction(self): return Transaction(self, None, False, False) + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + assert not terminated.is_set() + closed.set() + def terminate(self): terminated.set() @@ -166,11 +172,10 @@ async def execute(self, sql, *args): or (sql.startswith("COMMIT") and phase == "commit") ): reached.set() - # Transport termination, rather than a second task cancellation, - # releases a transaction that has already begun async cleanup. - while not terminated.is_set(): + # Public close must run independently of transaction unwinding. + while not closed.is_set(): try: - await terminated.wait() + await closed.wait() except asyncio.CancelledError: continue @@ -188,9 +193,9 @@ async def connect(): migrating.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(migrating, 1) - assert terminated.is_set() + assert closed.is_set() and not terminated.is_set() finally: - terminated.set() + closed.set() if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) From c601646a118940318e67f16f391695e56328e4e7 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:40:22 +0000 Subject: [PATCH 043/181] Close owned Postgres sessions before cancelling their work Send public driver cancellation before aborting the transport, within one absolute cleanup deadline. Separate borrowed migration execution so its operator keeps sole session supervision. Verify backend lock release and preserve interrupted migration errors. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + server/core/pg.py | 32 ++++ server/core/schema/pg_migrate.py | 52 +++++-- tests/integration/test_pg_online_index.py | 10 +- tests/integration/test_pg_owned_cleanup.py | 173 +++++++++++++++++++++ tests/unit/test_pg_close_owned.py | 144 +++++++++++++++++ tests/unit/test_pg_online_index.py | 21 ++- 7 files changed, 413 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_pg_owned_cleanup.py create mode 100644 tests/unit/test_pg_close_owned.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a132706be6..6e866c99a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,6 +897,7 @@ jobs: tests/integration/test_cell_tenancy_pg.py tests/integration/test_router_directory_pg.py tests/integration/test_pg_online_index.py + tests/integration/test_pg_owned_cleanup.py tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py diff --git a/server/core/pg.py b/server/core/pg.py index 576b1b1e4b..5daeb33a55 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -21,6 +21,7 @@ import asyncio import json import logging +import time import weakref from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager, suppress @@ -379,6 +380,37 @@ def _terminate(conn: Any) -> None: conn.terminate() +def _retrieve_task(task: asyncio.Task[Any]) -> None: + if not task.cancelled(): + task.exception() + + +async def close_owned(conn: Any, *, end: float) -> None: + """Close an owned session by an absolute time.monotonic deadline. + + Public close sends cancellation before transport cleanup can discard it. Abort + is a bounded fallback, not evidence that the backend has released its locks. + The owner preserves an earlier operation failure when this cleanup fails. + """ + task: asyncio.Task[Any] | None = None + try: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task = asyncio.create_task(conn.close(timeout=remaining)) + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + task.result() + except BaseException: + _terminate(conn) + if task is not None: + if not task.done(): + task.cancel() + task.add_done_callback(_retrieve_task) + raise + + async def _release(pool: Any, conn: Any) -> None: """Back to the pool within a bound: a statement still in flight (a client timeout, a cancelled request) has its cancel sent and acknowledged and the connection reset; diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index ebd230036b..371f61a449 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -19,6 +19,7 @@ import asyncio import hashlib import logging +import time from asyncio import sleep from collections.abc import Callable from contextlib import AbstractAsyncContextManager @@ -108,25 +109,54 @@ async def migrate( bounded (`lock_timeout`), so an `ALTER` queued behind a long transaction never holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" - files = migration_files(mig_dir) - # Validate every online declaration before even creating the ledger. - indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} - if any(index is not None and index.schema != schema for index in indexes.values()): - raise ValueError("online-index target differs from its ledger schema") + files, indexes = _validated(mig_dir, schema) async with (connect or pg.dedicated)() as conn: task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) try: return await asyncio.shield(task) except BaseException: - # Terminate before cancellation can enter asyncpg's unbounded rollback. - conn.terminate() - task.cancel() - await asyncio.wait({task}, timeout=1) - if task.done() and not task.cancelled(): - task.exception() + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. + pg._terminate(conn) raise +def _validated( + mig_dir: Path, schema: str +) -> tuple[list[Path], dict[str, pg_online_index.OnlineIndex | None]]: + files = migration_files(mig_dir) + # Validate every online declaration before even creating the ledger. + indexes = {f.name: pg_online_index.parse(f.read_text()) for f in files} + if any(index is not None and index.schema != schema for index in indexes.values()): + raise ValueError("online-index target differs from its ledger schema") + return files, indexes + + +async def migrate_borrowed( + conn: Any, + mig_dir: Path = PG_MIGRATIONS_DIR, + *, + schema: str = "loom", + lock_key: int = _LOCK_KEY, +) -> list[str]: + """Apply on the owner's session; its supervisor owns all failure cleanup. + + The caller retains its operation deadline and must close the session before + cancelling this work. Normal completion releases only the migration key. + """ + files, indexes = _validated(mig_dir, schema) + return await _migrate(conn, files, indexes, schema, lock_key) + + async def _lock(conn: Any, key: int) -> None: while True: set_query_tag("migrate.postgres.lock") diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index af93dd0937..1995e73bee 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -5,12 +5,14 @@ import asyncio import contextlib import os +import time from collections.abc import AsyncIterator from pathlib import Path from typing import Any import pytest +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.core.schema import pg_migrate, pg_online_index from server.router import directory @@ -315,7 +317,7 @@ async def direct_drop(): "DROP INDEX CONCURRENTLY loom_router.placements_by_cell_org" ) finally: - dropping.terminate() + await pg.close_owned(dropping, end=time.monotonic() + 1) dropping = asyncio.create_task(direct_drop()) try: @@ -325,6 +327,12 @@ async def direct_drop(): "AND wait_event IS NOT NULL)" ) await _until(conn, query) + # Cancel after catalog invalidation, not an earlier lock/I/O wait. + await _until( + conn, + "SELECT NOT indisvalid FROM pg_index " + "WHERE indexrelid='loom_router.placements_by_cell_org'::regclass", + ) dropping.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(dropping, 2) diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py new file mode 100644 index 0000000000..9085fce011 --- /dev/null +++ b/tests/integration/test_pg_owned_cleanup.py @@ -0,0 +1,173 @@ +"""Cancellation of owned Postgres work releases its actual backend locks.""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import time +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from server.core import pg +from server.core.schema import pg_migrate +from tests.integration._pg_database import create_database, drop_database + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def database() -> AsyncIterator[tuple[Any, Any]]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + observer = await asyncpg.connect(dsn) + try: + await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + yield conn, observer + finally: + await conn.close(timeout=1) + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def sleeping(observer: Any, pid: int) -> None: + async with asyncio.timeout(2): + for _ in range(400): + if await observer.fetchval( + "SELECT wait_event='PgSleep' FROM pg_stat_activity WHERE pid=$1", pid + ): + break + await asyncio.sleep(0.005) + else: + pytest.fail("backend did not reach pg_sleep") + assert await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + + +async def released(observer: Any, pid: int) -> None: + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +async def test_owned_migrator_cancellation_releases_server_locks(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await sleeping(observer, conn.get_server_pid()) + pid = conn.get_server_pid() + started = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - started < 1.5 + await released(observer, pid) + + +@pytest.mark.parametrize("transaction_context", [False, True]) +async def test_public_owned_close_releases_live_query_before_task_cancel( + database, transaction_context: bool +) -> None: + conn, observer = database + pid = conn.get_server_pid() + + async def work() -> None: + if transaction_context: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + else: + await conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_leaves_failure_cleanup_to_owner(database, tmp_path: Path) -> None: + conn, observer = database + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + task = asyncio.create_task(pg_migrate.migrate_borrowed(conn, tmp_path, schema="test")) + pid = conn.get_server_pid() + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_migrator_normal_completion_retains_owners_lock(database, tmp_path) -> None: + conn, observer = database + await conn.execute("SELECT pg_advisory_lock(712398)") + (tmp_path / "001_write.sql").write_text("INSERT INTO probe VALUES(2)") + assert await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") == ["001_write.sql"] + assert not conn.is_closed() + assert await observer.fetchval("SELECT count(*) FROM probe") == 2 + assert await conn.fetchval("SELECT pg_advisory_unlock(712398)") + assert ( + await conn.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=pg_backend_pid() AND locktype='advisory'" + ) + == 0 + ) + + +async def test_owned_close_interrupts_installed_transaction_rollback(database, monkeypatch) -> None: + import asyncpg + + conn, observer = database + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + + async def stalled_rollback(self, query, *args, **kwargs): + if self is conn and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + + async def work() -> None: + async with conn.transaction(): + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise ValueError("statement failure before rollback") + + task = asyncio.create_task(work()) + try: + await sleeping(observer, pid) + await pg.close_owned(conn, end=time.monotonic() + 0.5) + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_borrowed_failure_keeps_the_session_for_its_owner(database, tmp_path) -> None: + import asyncpg + + conn, _ = database + (tmp_path / "001_failure.sql").write_text("SELECT * FROM missing_table_for_failure") + with pytest.raises(asyncpg.UndefinedTableError): + await pg_migrate.migrate_borrowed(conn, tmp_path, schema="test") + assert not conn.is_closed() + assert await conn.fetchval("SELECT pg_advisory_unlock($1)", pg_migrate._LOCK_KEY) + assert await conn.fetchval("SELECT count(*) FROM test.pg_migrations") == 0 diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py new file mode 100644 index 0000000000..fbbe778a5d --- /dev/null +++ b/tests/unit/test_pg_close_owned.py @@ -0,0 +1,144 @@ +"""Owned-session cleanup delivers cancellation before aborting its transport.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Any + +import pytest + +from server.core import pg + +pytestmark = pytest.mark.asyncio + + +class Connection: + def __init__(self) -> None: + self.started = asyncio.Event() + self.finished = asyncio.Event() + self.aborted = False + self.timeouts: list[float] = [] + + async def close(self, *, timeout: float) -> None: # noqa: ASYNC109 — driver API. + self.timeouts.append(timeout) + self.started.set() + await self.finished.wait() + + def terminate(self) -> None: + self.aborted = True + self.finished.set() + + +async def test_public_close_completes_before_transport_abort() -> None: + conn = Connection() + conn.finished.set() + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.timeouts and 0 < conn.timeouts[0] <= 1 + assert not conn.aborted + + +async def test_expired_deadline_aborts_without_starting_close() -> None: + conn = Connection() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() - 1) + assert conn.aborted and not conn.timeouts + + +async def test_close_timeout_aborts_within_the_original_deadline() -> None: + conn = Connection() + started = time.monotonic() + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=started + 0.03) + assert conn.aborted and time.monotonic() - started < 0.3 + + +async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: + conn = Connection() + supervisor_started = asyncio.Event() + original = asyncio.CancelledError("operation cancelled") + + async def supervisor() -> None: + try: + supervisor_started.set() + await asyncio.Future() + except asyncio.CancelledError: + try: + await pg.close_owned(conn, end=time.monotonic() + 1) + except BaseException: # noqa: BLE001 — preserve the original cancellation. + pass + raise original from None + + task = asyncio.create_task(supervisor()) + await supervisor_started.wait() + task.cancel() + await asyncio.wait_for(conn.started.wait(), 1) + task.cancel() + with pytest.raises(asyncio.CancelledError, match="operation cancelled"): + await asyncio.wait_for(task, 0.3) + assert conn.aborted + + +async def test_failed_public_close_aborts_and_reports_failure() -> None: + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise ValueError("close failed") + + conn = Broken() + with pytest.raises(ValueError, match="close failed"): + await pg.close_owned(conn, end=time.monotonic() + 1) + assert conn.aborted + + +async def test_expired_close_child_exception_is_retrieved() -> None: + release = asyncio.Event() + errors: list[dict[str, Any]] = [] + + class LateFailure(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + try: + await release.wait() + except asyncio.CancelledError: + await release.wait() + raise ValueError("late close failure") + + conn = LateFailure() + loop = asyncio.get_running_loop() + old_handler = loop.get_exception_handler() + loop.set_exception_handler(lambda _, context: errors.append(context)) + try: + with pytest.raises(TimeoutError): + await pg.close_owned(conn, end=time.monotonic() + 0.01) + assert conn.aborted + release.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not errors + finally: + release.set() + loop.set_exception_handler(old_handler) + + +async def test_migration_failure_survives_cleanup_failure(monkeypatch, tmp_path) -> None: + import contextlib + + from server.core.schema import pg_migrate + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise RuntimeError("cleanup failed") + + conn = Broken() + + @contextlib.asynccontextmanager + async def connect(): + yield conn + + async def fail(*_: Any) -> None: + raise ValueError("migration failed") + + monkeypatch.setattr(pg_migrate, "_migrate", fail) + with pytest.raises(ValueError, match="migration failed"): + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert conn.aborted diff --git a/tests/unit/test_pg_online_index.py b/tests/unit/test_pg_online_index.py index 7fc43fa2bb..3bb0c19476 100644 --- a/tests/unit/test_pg_online_index.py +++ b/tests/unit/test_pg_online_index.py @@ -122,7 +122,7 @@ def never_connect(): @pytest.mark.asyncio @pytest.mark.parametrize("phase", ["statement", "rollback", "commit"]) -async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwind( +async def test_cancellation_closes_before_installed_asyncpg_transaction_unwind( tmp_path, phase ) -> None: import asyncio @@ -136,6 +136,7 @@ async def test_cancellation_terminates_before_installed_asyncpg_transaction_unwi (tmp_path / "001_one.sql").write_text("SELECT stalled_statement;") reached = asyncio.Event() terminated = asyncio.Event() + closed = asyncio.Event() class Connection: _pool_release_ctr = 0 @@ -143,11 +144,16 @@ class Connection: _protocol = SimpleNamespace(is_in_transaction=lambda: False) def is_closed(self): - return terminated.is_set() + return closed.is_set() or terminated.is_set() def transaction(self): return Transaction(self, None, False, False) + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + assert not terminated.is_set() + closed.set() + def terminate(self): terminated.set() @@ -166,11 +172,10 @@ async def execute(self, sql, *args): or (sql.startswith("COMMIT") and phase == "commit") ): reached.set() - # Transport termination, rather than a second task cancellation, - # releases a transaction that has already begun async cleanup. - while not terminated.is_set(): + # Public close must run independently of transaction unwinding. + while not closed.is_set(): try: - await terminated.wait() + await closed.wait() except asyncio.CancelledError: continue @@ -188,9 +193,9 @@ async def connect(): migrating.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(migrating, 1) - assert terminated.is_set() + assert closed.is_set() and not terminated.is_set() finally: - terminated.set() + closed.set() if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) From 51a8ff6498c3796d96c4c35946d4d70d8842d033 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:12:55 +0000 Subject: [PATCH 044/181] Close attach sessions before cancelling transaction work Supervise bounded Postgres phases independently, reserving cleanup inside each existing deadline. Send public close while query and transaction work are alive, then cancel and reap within that same bound. Borrowed checks and trigger DDL leave failed-session cleanup to their owner. Bound the lock owner's close and preserve its original failure across cleanup errors or repeated cancellation. Keep wait floors and normal busy retries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 9 +- docs/operations/cell-attach.md | 5 + server/cli/cell_attach.py | 37 +++- server/cli/cell_attach_pg.py | 160 +++++++++++------- server/cli/cell_prepare.py | 16 +- server/core/cell_triggers.py | 10 +- tests/integration/test_cell_attach_cleanup.py | 111 ++++++++++++ tests/unit/test_cell_attach.py | 53 ++++-- tests/unit/test_cell_attach_cleanup.py | 119 +++++++++++++ 10 files changed, 426 insertions(+), 95 deletions(-) create mode 100644 tests/integration/test_cell_attach_cleanup.py create mode 100644 tests/unit/test_cell_attach_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e866c99a5..3066d67299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -901,6 +901,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index dd1189b696..c738775756 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1067,7 +1067,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and - a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + a 5 s standby allowance. Each mode or trigger try reserves the smaller of one second + and one quarter of its original client bound for cleanup. Its independent owner + closes the session through the public driver API while query/transaction work is + still alive, then cancels and reaps that work within the same absolute deadline. + Borrowed checks and trigger DDL leave failed-session cleanup to that owner. Close + failure or exhausted cleanup time aborts the transport and preserves the original + failure; abort alone does not prove that server locks disappeared. A commit past + its deadline has an unknown outcome until the durable read. After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index e674da2997..13c65405f6 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -92,6 +92,11 @@ one try of each wait with no retry. The final heartbeat checks get at least two plus the router timeout, with one extension for a process first seen booting; this floor is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of it, since step 4 reads the same tables again. +Each Postgres mode or trigger try reserves cleanup time inside its client deadline. +On interruption its owner closes the session before cancelling transaction work. +A close that cannot complete uses transport abort and reports failure; that abort +does not establish that server locks or an unacknowledged commit have disappeared. +Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index bf4e81521e..2c3915ae66 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -187,14 +187,39 @@ async def open_lock(self) -> None: self._lock_cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) self.lock_conn = await self._lock_cm.__aenter__() + await apg.supervise( + self.lock_conn, self._take_lock(), end=time.monotonic() + _LOCK_STATEMENT_MS / 1000 + ) + + async def _take_lock(self) -> None: if refusals := await apg.direct_refusals(self.lock_conn, "attach"): raise Refused(refusals) await apg.take_lock(self.lock_conn, "attach") async def close_lock(self) -> None: + from server.core import pg + if self._lock_cm is not None: - await self._lock_cm.__aexit__(None, None, None) + cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + try: + if not closed: + await pg.close_owned(conn, end=end) + finally: + task = asyncio.create_task(cm.__aexit__(None, None, None)) + task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + task.cancel() + raise TimeoutError("attach lock cleanup deadline expired") + task.result() + except BaseException: + pg._terminate(conn) + task.cancel() + raise async def confirm_lock(self) -> None: """The lock is held still, or taken again (refusing when another run holds it now), @@ -631,6 +656,7 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 + failure: BaseException | None = None try: await self.open_lock() await self.read_own_row() @@ -660,8 +686,15 @@ async def run(self) -> int: answer = await self.serving() self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) return 0 + except BaseException as error: + failure = error + raise finally: - await self.close_lock() + try: + await self.close_lock() + except BaseException: + if failure is None: + raise async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 12e1808f8b..c71e1130c7 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -12,7 +12,7 @@ import asyncio import contextlib import time -from collections.abc import Awaitable, Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -89,6 +89,50 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True +@dataclass +class _Interrupted: + error: BaseException + + +async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: + try: + return await work + except (KeyboardInterrupt, SystemExit) as error: + # These exceptions otherwise stop asyncio's loop before the owner can close. + return _Interrupted(error) + + +async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: + """Bound owned work, reserving cleanup inside its original phase deadline. + + A failed borrowed transaction must remain alive until public close sends its + cancellation. Closing the transport first can leave its backend locks held. + """ + task = asyncio.create_task(_work_result(work)) + reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) + if not done: + raise TimeoutError("attach Postgres phase deadline expired") + result = task.result() + if isinstance(result, _Interrupted): + raise result.error + return result + except BaseException: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. + pg._terminate(conn) + raise + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -124,16 +168,11 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" identities = [] - try: - for _ in range(2): - tx = conn.transaction() - await tx.start() - identities.append(await conn.fetchrow(_IDENTITY)) - await tx.commit() - except BaseException: - # A cancelled read must not wait for a transaction context's rollback. - conn.terminate() - raise + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): @@ -537,30 +576,32 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S + return await supervise( + conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + ) + + +async def _trigger_try( + conn: Any, table: str, most_ms: int, wait: Wait, key: int +) -> cell_triggers.Outcome: + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: - async with asyncio.timeout(bound_s): - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") - try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) != "55P03": - raise - await conn.execute("RESET lock_timeout") - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) - await conn.execute("RESET lock_timeout") - outcome = await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) - return outcome - except BaseException: - # Closing the session releases its locks without asynchronous error cleanup. - conn.terminate() - raise + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, key)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, key) + return outcome def _busy_refusal(table: str, holders: list[Any]) -> Refusal: @@ -653,35 +694,32 @@ async def _mode_try(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> boo the standby allowance; False after a statement timeout rolls back, TimeoutError at the try's deadline.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S + return await supervise( + conn, _mode_transaction(conn, lock_ms, mark), end=time.monotonic() + bound_s + ) + + +async def _mode_transaction(conn: Any, lock_ms: int, mark: Callable[[str], None]) -> bool: + tx = conn.transaction() + await tx.start() try: - async with asyncio.timeout(bound_s): - tx = conn.transaction() - await tx.start() - try: - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] - ) - except Exception as e: - await tx.rollback() - if getattr(e, "sqlstate", None) in _TIMED_OUT: - return False - raise - await tx.commit() - return True - except BaseException: - # Termination is synchronous: cancellation never starts an unbounded rollback. - conn.terminate() + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + mark("unknown") + status = await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()") + if status != "UPDATE 1": + raise Refused([Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")]) + except Exception as e: + await tx.rollback() + if getattr(e, "sqlstate", None) in _TIMED_OUT: + return False raise + await tx.commit() + + return True async def end_mode_transactions(conn: Any, confirm_lock: Callable[[], Any]) -> None: diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4a820f652b..a71a541656 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -467,11 +467,17 @@ async def attach_lock() -> AsyncIterator[None]: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: try: - refusals = await cell_attach_pg.direct_refusals(conn, "prepare") - refusals += await cell_attach_pg.encoding_refusals(conn) - if refusals: - raise Stopped(refusals) - await cell_attach_pg.take_lock(conn, "prepare") + + async def take() -> None: + refusals = await cell_attach_pg.direct_refusals(conn, "prepare") + refusals += await cell_attach_pg.encoding_refusals(conn) + if refusals: + raise Stopped(refusals) + await cell_attach_pg.take_lock(conn, "prepare") + + await cell_attach_pg.supervise( + conn, take(), end=time.monotonic() + _LOCK_STATEMENT_MS / 1000 + ) except Stopped as e: raise Refused(str(e)) from None yield diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 1899a46bab..68e74777ec 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -324,7 +324,8 @@ async def _run_ddl( two causes cells.md names kept it from applying (`busy`: its lock not taken, with the sessions it waited behind; `denied`: a table this role cannot alter), which a boot serves past; any other error fails it, as a failed read of the state does. `warn` logs - a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`). + The borrowing caller owns failed-session cleanup, including cancellation/rollback.""" import asyncpg for attempt in range(attempts): @@ -344,7 +345,6 @@ async def _run_ddl( return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: if not rolled_back: - conn.terminate() raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e @@ -352,7 +352,6 @@ async def _run_ddl( return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: if not rolled_back: - conn.terminate() raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -368,11 +367,6 @@ async def _run_ddl( held, ) return Outcome("busy", held) - except BaseException: - if not rolled_back: - # Cancellation must not wait for another statement on this session. - conn.terminate() - raise return Outcome("busy") diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..145be1188d --- /dev/null +++ b/tests/integration/test_cell_attach_cleanup.py @@ -0,0 +1,111 @@ +"""Attach owners close active sessions before cancelling transaction work.""" + +import argparse +import asyncio +import contextlib +import time + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import cell_triggers, pg +from tests.integration.test_pg_owned_cleanup import database as database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize( + "owner,phase", + [ + ("trigger", "query"), + ("trigger", "rollback"), + ("mode", "query"), + ("mode", "rollback"), + ("direct", "query"), + ], +) +@pytest.mark.parametrize("interruption", ["cancel", "deadline", "second-cancel"]) +async def test_attach_owner_releases_backend_locks_before_return( + database, monkeypatch, owner, phase, interruption +): + import asyncpg + + conn, observer = database + await conn.execute( + "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" + ) + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + closing = asyncio.Event() + close = asyncpg.Connection.close + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query.startswith("SET LOCAL statement_timeout"): + return await execute(self, "SET LOCAL statement_timeout = '10000ms'") + if query == "RAISE Python error": + raise ValueError("trigger statement failed") + if phase == "rollback" and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + if owner == "mode" and query.startswith("LOCK TABLE"): + await execute(self, "SELECT * FROM probe FOR UPDATE") + if phase == "rollback": + raise ValueError("mode statement failed") + return await execute(self, "SELECT pg_sleep(10)") + if owner == "direct" and query.startswith("BEGIN"): + await execute(self, query) + return await execute(self, "SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + async def closing_session(self, *args, **kwargs): + was_open = not self.is_closed() + await close(self, *args, **kwargs) + if self is conn and was_open and interruption == "second-cancel": + closing.set() + await asyncio.Future() + + async def ensure(conn, table, **kwargs): + statements = ["SELECT * FROM probe FOR UPDATE"] + statements += ["RAISE Python error"] if phase == "rollback" else ["SELECT pg_sleep(10)"] + return await cell_triggers._run_ddl( + conn, table, statements, kwargs["lock_timeout"], 1, warn=False + ) + + @contextlib.asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + monkeypatch.setattr(asyncpg.Connection, "close", closing_session) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure) + monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) + monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) + monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) + monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) + monkeypatch.setattr(pg, "dedicated", dedicated) + if owner == "trigger": + work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + elif owner == "mode": + work = apg._mode_try(conn, 100, lambda _: None) + else: + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + work = run.open_lock() + task = asyncio.create_task(work) + try: + await sleeping(observer, pid) + started = time.monotonic() + if interruption != "deadline": + task.cancel() + if interruption == "second-cancel": + await asyncio.wait_for(closing.wait(), 1) + task.cancel() + with pytest.raises(TimeoutError if interruption == "deadline" else asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 1.2) + assert time.monotonic() - started < 0.7 + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0ecd3a96b3..05e15b10c2 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,6 +8,7 @@ import argparse import asyncio import json +import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -496,6 +497,24 @@ async def __aexit__(self, *exc: Any) -> None: def terminate(self) -> None: self.executed.append("TERMINATE") + async def close(self, **kwargs: Any) -> None: + assert kwargs["timeout"] > 0 + self.executed.append("CLOSE") + + def is_closed(self) -> bool: + return bool(self.executed and self.executed[-1] in {"CLOSE", "TERMINATE"}) + + +def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + supervise = apg.supervise + + async def bounded(conn: Any, work: Any, *, end: float) -> Any: + if bounds is not None: + bounds.append(end - time.monotonic()) + return await supervise(conn, work, end=time.monotonic() + 0.01) + + monkeypatch.setattr(apg, "supervise", bounded) + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -750,13 +769,7 @@ async def execute(self, sql: str, *args: Any) -> str: conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] - timeout = asyncio.timeout - - def bounded(seconds: float) -> Any: - bounds.append(seconds) - return timeout(0) - - monkeypatch.setattr(apg.asyncio, "timeout", bounded) + _short_phases(monkeypatch, bounds) marks: list[str] = [] async def confirm() -> None: @@ -773,8 +786,9 @@ async def ready() -> list[str]: timeout=0.2, ) assert "COMMIT" in conn.executed - assert conn.executed[-1] == "TERMINATE" - assert bounds == [bound_s] + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert bounds == pytest.approx([bound_s], abs=0.001) assert marks == ["unknown"] @@ -793,8 +807,7 @@ async def execute(self, sql: str, *args: Any) -> str: return result conn = _StalledRollback() - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task(apg._mode_try(conn, 100, marks.append)) done, _ = await asyncio.wait([task], timeout=0.2) @@ -804,7 +817,8 @@ async def execute(self, sql: str, *args: Any) -> str: await task assert task in done, "mode deadline is stuck awaiting rollback" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == (["unknown"] if stall == "UPDATE" else []) @@ -846,7 +860,8 @@ async def execute(self, sql: str, *args: Any) -> str: assert task.cancelled() else: assert isinstance(task.exception(), ValueError) - assert conn.executed[-1] == "TERMINATE" + assert "CLOSE" not in conn.executed + assert "TERMINATE" not in conn.executed assert "ROLLBACK" not in conn.executed @@ -865,7 +880,8 @@ async def ready() -> list[str]: await apg.commit_enforced( conn, confirm, ready, apg.Wait(2, clock), lambda _t: None, marks.append ) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == [] @@ -1069,8 +1085,7 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False ) - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) @@ -1078,7 +1093,8 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: try: assert task in done, f"trigger deadline is stuck at {stall}" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed if stall in {"migration", "ddl", "commit", "rollback"}: assert apg._MIGRATION_UNLOCK not in conn.executed finally: @@ -1110,7 +1126,8 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..20e17ac786 --- /dev/null +++ b/tests/unit/test_cell_attach_cleanup.py @@ -0,0 +1,119 @@ +"""The attach owner preserves failures through bounded public-close cleanup.""" + +import argparse +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import pg + + +@pytest.mark.asyncio +async def test_failed_work_keeps_its_error_when_public_close_also_fails(): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise RuntimeError("close failed") + + def terminate(self): + events.append("abort") + + async def work(): + raise ValueError("original work failure") + + with pytest.raises(ValueError, match="original work failure"): + await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) + assert events == ["close", "abort"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_second_cancellation_preserves_original_failure_and_closes_before_cancel(failed): + entered, closing = asyncio.Event(), asyncio.Event() + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + closing.set() + await asyncio.Future() + + def terminate(self): + events.append("abort") + + async def work(): + entered.set() + try: + if failed: + raise ValueError("original work failure") + await asyncio.Future() + finally: + events.append("work ended") + + task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + await entered.wait() + if not failed: + task.cancel("first cancellation") + await closing.wait() + task.cancel("second cancellation") + with pytest.raises( + ValueError if failed else asyncio.CancelledError, + match="original work failure" if failed else "first cancellation", + ): + await task + assert events.count("abort") >= 1 + if not failed: + assert events.index("close") < events.index("work ended") + + +@pytest.mark.asyncio +async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + cm = AsyncMock() + exit_started = asyncio.Event() + + async def exit(*_): + exit_started.set() + await asyncio.Future() + + cm.__aexit__.side_effect = exit + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + run._lock_cm, run.lock_conn = cm, conn + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + started = time.monotonic() + with pytest.raises(TimeoutError): + await run.close_lock() + assert time.monotonic() - started < 0.15 + assert exit_started.is_set() and run.lock_conn is None + conn.close.assert_awaited_once() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): + router = AsyncMock() + router.call.return_value = {"state": "attaching"} + run = cell_attach.Run( + None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None + ) + run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.dry_run = AsyncMock(return_value=0) + run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) + try: + raise LookupError("unrelated caller exception") + except LookupError: + with pytest.raises( + ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" + ): + await run.run() From 1f69c41935eec5259229393806bdd46bc8d6f758 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:15:27 +0000 Subject: [PATCH 045/181] Supervise native cell control cleanup before cancelling work Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 + docs/operations/cell-control.md | 6 +- server/cli/cell.py | 8 +- server/cli/cell_control.py | 215 +++++++++++++---- server/cli/cell_prepare.py | 38 ++- server/core/pg.py | 27 ++- .../integration/test_cell_control_cleanup.py | 216 ++++++++++++++++++ tests/unit/test_cell_attach.py | 3 + tests/unit/test_cell_control.py | 57 +++-- tests/unit/test_cell_prepare.py | 16 +- 10 files changed, 487 insertions(+), 108 deletions(-) create mode 100644 tests/integration/test_cell_control_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 59ed6fef64..d5f24fdabc 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1248,6 +1248,15 @@ cleanup have explicit bounds; an interrupted or uncertain local transaction neve continues on another connection. No operation record, persistent census, phase or lease is added. The UUID is coordination metadata and holds no memory content. +One ephemeral owner supervises the command's raw session and child work. Opening keeps +its 75-second bound and each local control transaction keeps its 60-second bound, +reserving cleanup time inside that bound. On failure it sends public connection close +before cancelling child work, so active statements and rollback cannot prevent bounded +cleanup. An unexpected transaction error propagates immediately; it never waits for +rollback on a session the owner will discard. Uncapped command work retains the existing +one-second session-exit allowance, which cannot extend an expired bounded phase. A +transport abort remains an unknown outcome, not proof that the server backend ended. + The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: diff --git a/docs/operations/cell-control.md b/docs/operations/cell-control.md index b606d9aaf9..30892abf7f 100644 --- a/docs/operations/cell-control.md +++ b/docs/operations/cell-control.md @@ -89,9 +89,11 @@ async def main(): async def call(path, body): return await cell_control.request(url, settings.cell, None, path, body) - async with cell_control.session("state cleanup") as conn: + async def cleanup(owner): row = await cell_control.request(url, settings.cell, None, "", method="GET") - await cell_control.backout(conn, call, row) + await cell_control.backout(owner, call, row) + + await cell_control.run("state cleanup", cleanup) finally: await ch.disconnect() diff --git a/server/cli/cell.py b/server/cli/cell.py index 77b13e8aec..e43a5a4d07 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -366,7 +366,8 @@ async def _remove_locked( from server.cli import cell_control from server.core import cell - async with cell_control.session("remove") as conn: + async def remove(owner: cell_control.Owner) -> dict[str, Any]: + conn = owner.conn async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: return await cell_control.request(router, args.cell, transport, path, body) @@ -377,8 +378,7 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: raise cell_control._refused(reason) await cell_control.paired(conn, row) if row["is_first"]: - async with cell_control.transaction(conn): - state, _ = await cell.read_state_if_present(conn) + state, _ = await owner.transaction(lambda: cell.read_state_if_present(conn)) if state != "open": raise cell_control._refused( f"the first cell's Postgres reads {state}: run the full-state cleanup " @@ -390,6 +390,8 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: router, args.cell, transport, "", cell_control.preconditions(row), method="DELETE" ) + return await cell_control.run("remove", remove) + def _cmd_which(args: argparse.Namespace) -> int: router = parse_router(args) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 3c03a75e2a..84661747fd 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -4,14 +4,16 @@ import asyncio import json -from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import asynccontextmanager -from typing import Any +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any, cast from uuid import UUID from server.cli import cell_attach_pg as apg from server.cli.cell_ch import Refusal from server.cli.cell_ch import Refused as Refused +from server.core import pg from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings @@ -95,53 +97,164 @@ async def mutate( return answer -@asynccontextmanager -async def transaction(conn: Any) -> AsyncIterator[Any]: - """Check, write and durably commit on the lock backend; cancellation ends ownership.""" - settled = False - tx = conn.transaction() +@dataclass(frozen=True) +class _Fatal: + error: SystemExit | KeyboardInterrupt + + +class _ForwardFatal(BaseException): + def __init__(self, fatal: _Fatal) -> None: + self.fatal = fatal + + +async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: try: - async with asyncio.timeout(TRANSACTION_S): - await tx.start() + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _Fatal(error) + except _ForwardFatal as error: + return error.fatal + + +class Owner: + """One raw backend and one cleanup owner for a native control command.""" + + def __init__(self) -> None: + self.conn: Any = None + self.phase_end: float | None = None + self.close_end: float | None = None + self.closed = False + self._close_task: asyncio.Task[Any] | None = None + self._tasks: set[asyncio.Task[Any]] = set() + + def _start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _Fatal]: + task = asyncio.create_task(_captured(work)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + task.add_done_callback(pg._retrieve_task) + return task + + async def _open(self, what: str) -> None: + self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) + if self.closed: + # A connector returning after cancellation cannot publish a usable session. + await pg.close_owned(self.conn, end=self.close_end or 0) + raise _refused("connection arrived after the control attempt ended") + try: + await pg._check_deployment(self.conn) + await pg.init_connection(self.conn) + except Exception as error: + pg._raise_if_unavailable(error) + raise + refused = await apg.direct_refusals(self.conn, what) + refused += await apg.encoding_refusals(self.conn) + if refused: + raise Refused(refused) + await apg.take_lock(self.conn, what) + + async def close(self) -> None: + if not self.closed: + self.closed = True + self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + if self.conn is None: + return + if self._close_task is None: + self._close_task = asyncio.create_task( + _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) + ) + self._close_task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait( + {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) + ) + if not done: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + result = self._close_task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: + pg._terminate(self.conn) + if not self._close_task.done(): + self._close_task.cancel() + raise + + async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: + for task in tasks: + if not task.done(): + task.cancel() + try: + if tasks: + await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) + except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. + if self.conn is not None: + pg._terminate(self.conn) + + async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: + if self.closed or self.phase_end is not None: + raise _refused("a closed or active control phase cannot start another phase") + self.phase_end = time.monotonic() + total_s + task = self._start(work) + try: + done, _ = await asyncio.wait( + {task}, + timeout=max(0, self.phase_end - min(1.0, total_s / 4) - time.monotonic()), + ) + if not done: + raise TimeoutError( + "cell control phase deadline expired; outcome unknown; stop and rerun" + ) + result = task.result() + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException: try: - await conn.execute("SET LOCAL lock_timeout = '5s'") - await conn.execute("SET LOCAL statement_timeout = '10s'") - await check_lock(conn) - await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") - yield conn - except Exception: - await tx.rollback() - settled = True - raise - else: - await tx.commit() - settled = True - finally: - if not settled: - # This includes cancellation during COMMIT/rollback. A later session is a new run. - conn.terminate() + await self.close() + except BaseException: # noqa: BLE001 — keep the original phase failure/cancellation. + pass + await self._stop({task}) + raise + self.phase_end = None + return result + async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: + return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) -@asynccontextmanager -async def session(what: str) -> AsyncIterator[Any]: - """A direct, marked, UTF8 session holding ATTACH for its entire control run.""" - from server.core import pg - cm = pg.dedicated(statement_timeout_ms=60_000) - conn = None +async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: + """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + owner = Owner() + try: + await owner.call(lambda: owner._open(what), total_s=OPEN_S) + result = await asyncio.shield(owner._start(lambda: work(owner))) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await owner.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await owner._stop(set(owner._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise try: - async with asyncio.timeout(OPEN_S): - conn = await cm.__aenter__() - refused = await apg.direct_refusals(conn, what) - refused += await apg.encoding_refusals(conn) - if refused: - raise Refused(refused) - await apg.take_lock(conn, what) - yield conn + await owner.close() finally: - if conn is not None: - # Use dedicated()'s bounded close; this does not prove the server backend ended. - await cm.__aexit__(None, None, None) + await owner._stop(set(owner._tasks)) + return result + + +async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: + """Borrow the owner's backend; every failure discards it without a rollback wait.""" + tx = conn.transaction() + await tx.start() + await conn.execute("SET LOCAL lock_timeout = '5s'") + await conn.execute("SET LOCAL statement_timeout = '10s'") + await check_lock(conn) + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + result = await work() + await tx.commit() + return result async def request( @@ -196,12 +309,12 @@ async def _acceptance(conn: Any) -> dict[str, Any]: } -async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: +async def backout(owner: Owner, call: Call, row: dict[str, Any]) -> None: """The runbook's exact state cleanup, fenced against queued controllers/placements.""" from server.core.schema import pg_migrate - async with transaction(conn): - before = await _acceptance(conn) + conn = owner.conn + before = await owner.transaction(lambda: _acceptance(conn)) first = before["first"] if first is not None and (first["cell"], first["is_first"]) != (row["cell"], row["is_first"]): raise _refused("the local first row names another registration") @@ -218,7 +331,8 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: limited = await mutate(conn, call, row, "/limit", {"closed": True}) if type(limited.get("placed_orgs")) is not int or limited["placed_orgs"] != 0: raise _refused("cell holds placements: cleanup refused; the cell remains closed") - async with transaction(conn): + + async def clear_state() -> None: await paired(conn, row) if json.dumps(await _acceptance(conn), sort_keys=True) != json.dumps( before, sort_keys=True @@ -232,9 +346,12 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: await conn.execute( "ALTER TABLE loom.cell_first ENABLE ALWAYS TRIGGER cell_first_no_truncate" ) + + await owner.transaction(clear_state) + # Once state is open, a migrating boot cannot recreate the org triggers. The same # session still holds ATTACH; a retry repeats the durable state cleanup first. - async with transaction(conn): + async def clear_triggers() -> None: await conn.execute("SELECT pg_advisory_xact_lock($1)", pg_migrate._LOCK_KEY) triggers = await conn.fetch( "SELECT t.tgname, c.relname FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid " @@ -244,3 +361,5 @@ async def backout(conn: Any, call: Call, row: dict[str, Any]) -> None: for trigger in triggers: name, table = (str(trigger[k]).replace('"', '""') for k in ("tgname", "relname")) await conn.execute(f'DROP TRIGGER "{name}" ON loom."{table}"') + + await owner.transaction(clear_triggers) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index c4ac332757..6e94ae1058 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,13 +26,11 @@ import argparse import asyncio -import contextlib import hashlib import json import secrets import sys import time -from collections.abc import AsyncIterator from dataclasses import dataclass from typing import Any @@ -388,7 +386,8 @@ async def _write_rows( from `current` and `nxt`, in one transaction; the rows written.""" is_first = bool(answer.get("is_first")) written = [] - async with cell_control.transaction(conn): + + async def write() -> list[str]: # A state CAS does not need to count every tenancy row again. fresh = await _read_rows(conn, count_tenancy=False) before = (rows.deployment_id, rows.first, rows.mode, rows.current, rows.next) @@ -418,7 +417,9 @@ async def _write_rows( nxt, ) written.append("own") - return written + return written + + return await cell_control.transaction(conn, write) def _answer_problem(rows: _CellRows, answer: dict[str, Any], beats: list[_Beat]) -> str: @@ -457,19 +458,13 @@ async def prepare( router: httpx.URL, transport: httpx.AsyncBaseTransport | None = None, ) -> int: - """Prepare the cell, holding the attach lock for the run (`attach_lock`).""" - async with attach_lock() as conn: - return await _prepare_locked(ch, args, router, transport, conn) - - -@contextlib.asynccontextmanager -async def attach_lock() -> AsyncIterator[Any]: - """Hand the exact lock backend to every acceptance-state read and write.""" + """Prepare on one independently supervised ATTACH-lock backend.""" try: - async with cell_control.session("prepare") as conn: - yield conn - except cell_control.Refused as e: - raise Refused(str(e)) from None + return await cell_control.run( + "prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner) + ) + except cell_control.Refused as error: + raise Refused(str(error)) from None async def _prepare_locked( @@ -477,10 +472,10 @@ async def _prepare_locked( args: argparse.Namespace, router: httpx.URL, transport: httpx.AsyncBaseTransport | None, - conn: Any, + owner: cell_control.Owner, ) -> int: - async with cell_control.transaction(conn): - rows = await _read_rows(conn) + conn = owner.conn + rows = await owner.transaction(lambda: _read_rows(conn)) if not rows.deployment_id: raise Refused("this cell's Postgres records no deployment id: start the cell once first") if rows.first is not None and rows.first[1] != settings.cell: @@ -527,7 +522,10 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: if problem := _answer_problem(rows, snapshot, beats): raise Refused(problem) current, nxt = _own_rows(rows, snapshot, every) - written = await _write_rows(conn, rows, answer, current, nxt) + written = await owner.call( + lambda: _write_rows(conn, rows, answer, current, nxt), + total_s=cell_control.TRANSACTION_S, + ) print( json.dumps( { diff --git a/server/core/pg.py b/server/core/pg.py index 5daeb33a55..d7bdf5b3e4 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -462,20 +462,13 @@ def may_have_committed(e: BaseException) -> bool: return isinstance(e, PostgresUnavailableError) and bool(e.extra.get("outcome_unknown")) -@asynccontextmanager -async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: - """A connection of its own, outside the pool, with no client timeout and no server - statement timeout unless `statement_timeout_ms` asks for one: for work that runs as - long as it takes (migrations, an erasure's deletes). - A CLI process's first one checks the deployment (`expect_deployment`). - `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps - it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session - without it).""" +async def _connect_dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> Any: + """Return the raw dedicated session before its owner initializes it.""" import asyncpg _require_dsn() try: - conn = await asyncpg.connect( + return await asyncpg.connect( settings.pg_dsn, command_timeout=None, server_settings={ @@ -492,6 +485,20 @@ async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> As except Exception as e: _raise_if_unavailable(e) raise + + +@asynccontextmanager +async def dedicated(*, statement_timeout_ms: int = 0, marked: bool = True) -> AsyncIterator[Any]: + """A connection of its own, outside the pool, with no client timeout and no server + statement timeout unless `statement_timeout_ms` asks for one: for work that runs as + long as it takes (migrations, an erasure's deletes). + A CLI process's first one checks the deployment (`expect_deployment`). + `statement_timeout_ms` bounds its statements as a startup value (a RESET ALL keeps + it); `marked` False leaves the tenancy marker out (`loom doctor` reads a session + without it).""" + import asyncpg + + conn = await _connect_dedicated(statement_timeout_ms=statement_timeout_ms, marked=marked) # Opening includes the deployment check and the codecs (jsonb reads back as a dict, as # the pool's): an outage there is the 503; the block's own errors pass through as # they are (an erasure's deadlock reaches its retry). diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py new file mode 100644 index 0000000000..2a37ca6170 --- /dev/null +++ b/tests/integration/test_cell_control_cleanup.py @@ -0,0 +1,216 @@ +"""Native cell-control interruption closes its actual Postgres backend and locks.""" + +from __future__ import annotations + +import argparse +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any + +import httpx +import pytest + +from server.cli import cell_control, cell_prepare +from server.core import pg +from server.settings import settings +from tests.integration._pg_database import own_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def native_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + async with own_database() as dsn: + monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(pg, "_expected", None) + observer = await asyncpg.connect(dsn) + await observer.execute( + "CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)" + ) + try: + yield observer + finally: + await observer.close(timeout=1) + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_native_prepare_interrupt_releases_backend_and_locks( + native_database, monkeypatch, interrupt +): + observed = asyncio.Event() + backend: dict[str, Any] = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + if interrupt == "deadline": + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + await asyncio.wait_for(observed.wait(), 3) + await sleeping(native_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("native prepare cancelled") + with pytest.raises(asyncio.CancelledError if interrupt == "cancel" else TimeoutError): + await asyncio.wait_for(task, 1.5) + assert time.monotonic() - began < 1.5 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_body_failure_is_not_hidden_by_rollback(native_database, monkeypatch): + import asyncpg + + execute = asyncpg.Connection.execute + original = ValueError("original native transaction failure") + backend = {} + + async def read(conn, **_): + backend["pid"] = conn.get_server_pid() + await conn.execute("SELECT * FROM probe FOR UPDATE") + raise original + + async def stalled_rollback(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) + began = time.monotonic() + with pytest.raises(ValueError, match="original native transaction failure") as captured: + await asyncio.wait_for( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")), 1.5 + ) + assert captured.value is original + assert time.monotonic() - began < 0.4 + await released(native_database, backend["pid"]) + + +async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( + native_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + + async def read(conn, **_): + backend["conn"], backend["pid"] = conn, conn.get_server_pid() + observed.set() + await conn.execute("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + async def delayed_close(self, **kwargs): + await real_close(self, **kwargs) + if self is backend.get("conn"): + close_started.set() + await close_return.wait() + + monkeypatch.setattr(cell_prepare, "_read_rows", read) + monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) + task = asyncio.create_task( + cell_prepare.prepare(None, argparse.Namespace(), httpx.URL("http://router")) + ) + try: + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("first native cancellation") + await asyncio.wait_for(close_started.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(asyncio.CancelledError, match="first native cancellation"): + await asyncio.wait_for(task, 0.5) + await released(native_database, backend["pid"]) + finally: + close_return.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["open", "phase", "body"]) +@pytest.mark.parametrize("kind", ["org", "keyboard"]) +async def test_native_fatal_guard_is_inert_until_backend_closed(native_database, stage, kind): + import os + import sys + + # A separate interpreter keeps a regression in child fatal-exception transport + # from stopping pytest's event loop. Proof runs before the native command exits. + from tests.integration._pg_database import with_database + + base = os.environ["LOOM_PG_DSN"] + dsn = with_database(base, await native_database.fetchval("SELECT current_database()")) + script = """ +import argparse, asyncio, json, os, sys +import asyncpg, httpx +from server.cli import cell_control, cell_prepare +from server.cli._ch import OrgNotHeld +from server.core import pg +from server.settings import settings +stage, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = OrgNotHeld('org refused') if kind == 'org' else KeyboardInterrupt('guard interrupted') +backend = {} +real_init = pg.init_connection +async def initialize(conn): + await real_init(conn) + backend['pid'] = conn.get_server_pid() + if stage == 'open': + raise original +async def read(conn, **kwargs): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'phase': + raise original + return cell_prepare._CellRows('dep-first', None, '', None, None, 0) +async def beats(ch): + raise original +pg.init_connection = initialize +cell_prepare._read_rows = read +cell_prepare._read_beats = beats +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: + await cell_prepare.prepare(None, argparse.Namespace(), httpx.URL('http://router')) + except BaseException as found: + assert found is original + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print(json.dumps({'original': True, 'backend_absent': True})) + else: + raise AssertionError('fatal guard became success') + finally: + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, _stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0, "native fatal exception interrupted its supervisor" + assert b'"backend_absent": true' in stdout + finally: + if child.returncode is None: + child.kill() + await child.wait() diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index d573d37423..65feba889d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -518,6 +518,9 @@ async def __aexit__(self, *exc: Any) -> None: return _Tx() + async def close(self, *, timeout: float) -> None: + self.executed.append("CLOSE") + def terminate(self) -> None: self.executed.append("TERMINATE") diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 71d687baf9..74ac1c130d 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -14,45 +14,58 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("stall", ["COMMIT", "ROLLBACK"]) -async def test_uncertain_transaction_ends_owned_session_without_unbounded_cleanup( - monkeypatch, stall -): +async def test_uncertain_commit_closes_before_cancelling_work(monkeypatch): class Stalled(FakeConn): async def execute(self, sql: str, *args: Any) -> str: answer = await super().execute(sql, *args) - if sql == stall: - await asyncio.Future() + if sql == "COMMIT": + try: + await asyncio.Future() + finally: + self.executed.append("WORK ENDED") return answer conn = Stalled() - monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.01) + owner = cell_control.Owner() + owner.conn = conn + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) with pytest.raises(TimeoutError): - async with cell_control.transaction(conn): - if stall == "ROLLBACK": - raise ValueError("refuse") - assert conn.executed[-1] == "TERMINATE" - assert stall in conn.executed + await owner.transaction(AsyncMock()) + assert conn.executed.index("CLOSE") < conn.executed.index("WORK ENDED") + assert "TERMINATE" not in conn.executed and "ROLLBACK" not in conn.executed assert any("pg_logical_emit_message" in sql for sql in conn.executed) @pytest.mark.asyncio -async def test_cancellation_during_body_never_waits_for_rollback(): +async def test_failure_never_starts_rollback_or_replaces_original(): conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn + original = ValueError("original failure") + with pytest.raises(ValueError) as found: + await owner.transaction(AsyncMock(side_effect=original)) + assert found.value is original + assert conn.executed[-1] == "CLOSE" + assert "ROLLBACK" not in conn.executed and "TERMINATE" not in conn.executed + + +@pytest.mark.asyncio +async def test_cancellation_during_body_closes_before_work_cancel(): + conn = FakeConn() + owner = cell_control.Owner() + owner.conn = conn entered = asyncio.Event() - async def work(): - async with cell_control.transaction(conn): - entered.set() - await asyncio.Future() + async def body(): + entered.set() + await asyncio.Future() - task = asyncio.create_task(work()) + task = asyncio.create_task(owner.transaction(body)) await entered.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): + task.cancel("original cancellation") + with pytest.raises(asyncio.CancelledError, match="original cancellation"): await task - assert conn.executed[-1] == "TERMINATE" - assert "ROLLBACK" not in conn.executed + assert conn.executed[-1] == "CLOSE" and "ROLLBACK" not in conn.executed def snapshot(): diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index 3071dddccf..58040a584d 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse -import contextlib import json from datetime import UTC, datetime from typing import Any @@ -31,7 +30,19 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). - monkeypatch.setattr(cell_prepare, "attach_lock", contextlib.nullcontext) + class Owner: + conn = None + + async def transaction(self, work): + return await work() + + async def call(self, work, **_): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) def _args(**over: Any) -> argparse.Namespace: @@ -395,7 +406,6 @@ async def _write(_conn: Any, _rows: Any, _answer: Any, current: Any, nxt: Any) - ("_write_rows", _write), ): monkeypatch.setattr(cell_prepare, name, fake) - monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) directory_mode = ["recording"] From 5f76257d3d34ca983958c36446b265e7579fa8bc Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:12:55 +0000 Subject: [PATCH 046/181] Close attach sessions before cancelling transaction work Supervise bounded Postgres phases independently, reserving cleanup inside each existing deadline. Send public close while query and transaction work are alive, then cancel and reap within that same bound. Borrowed checks and trigger DDL leave failed-session cleanup to their owner. Bound the lock owner's close and preserve its original failure across cleanup errors or repeated cancellation. Keep wait floors and normal busy retries. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 9 +- docs/operations/cell-attach.md | 5 + server/cli/cell_attach.py | 32 ++++- server/cli/cell_attach_pg.py | 109 +++++++++++----- server/core/cell_triggers.py | 10 +- tests/integration/test_cell_attach_cleanup.py | 111 ++++++++++++++++ tests/unit/test_cell_attach.py | 53 +++++--- tests/unit/test_cell_attach_cleanup.py | 119 ++++++++++++++++++ 9 files changed, 386 insertions(+), 63 deletions(-) create mode 100644 tests/integration/test_cell_attach_cleanup.py create mode 100644 tests/unit/test_cell_attach_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e866c99a5..3066d67299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -901,6 +901,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index d5f24fdabc..70a203783a 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1067,7 +1067,14 @@ Then, in order (the step numbers are the ones the rest of this page uses): lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and - a 5 s standby allowance. A commit past it has an unknown outcome until the durable read. + a 5 s standby allowance. Each mode or trigger try reserves the smaller of one second + and one quarter of its original client bound for cleanup. Its independent owner + closes the session through the public driver API while query/transaction work is + still alive, then cancels and reaps that work within the same absolute deadline. + Borrowed checks and trigger DDL leave failed-session cleanup to that owner. Close + failure or exhausted cleanup time aborts the transport and preserves the original + failure; abort alone does not prove that server locks disappeared. A commit past + its deadline has an unknown outcome until the durable read. After a failed try, the mode transaction an earlier run left in flight is found by its lock (only attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) and ended. From the commit on the mode only rises: every later refusal leaves the cell diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index e674da2997..13c65405f6 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -92,6 +92,11 @@ one try of each wait with no retry. The final heartbeat checks get at least two plus the router timeout, with one extension for a process first seen booting; this floor is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of it, since step 4 reads the same tables again. +Each Postgres mode or trigger try reserves cleanup time inside its client deadline. +On interruption its owner closes the session before cancelling transaction work. +A close that cannot complete uses transport abort and reports failure; that abort +does not establish that server locks or an unacknowledged commit have disappeared. +Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index b3cb44d9cf..ed57bb88da 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -170,9 +170,29 @@ async def open_lock(self) -> None: self.lock_conn = await self._lock_cm.__aenter__() async def close_lock(self) -> None: + from server.core import pg + if self._lock_cm is not None: - await self._lock_cm.__aexit__(None, None, None) + cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + try: + if not closed: + await pg.close_owned(conn, end=end) + finally: + task = asyncio.create_task(cm.__aexit__(None, None, None)) + task.add_done_callback(pg._retrieve_task) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + task.cancel() + raise TimeoutError("attach lock cleanup deadline expired") + task.result() + except BaseException: + pg._terminate(conn) + task.cancel() + raise async def confirm_lock(self) -> None: """A lost backend ends the owner; a new run must claim and start again.""" @@ -588,6 +608,7 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 + failure: BaseException | None = None try: await self.open_lock() await self.read_own_row() @@ -616,8 +637,15 @@ async def run(self) -> int: answer = await self.serving() self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) return 0 + except BaseException as error: + failure = error + raise finally: - await self.close_lock() + try: + await self.close_lock() + except BaseException: + if failure is None: + raise async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 8ae6436f3f..d24f749996 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -13,7 +13,7 @@ import contextlib import json import time -from collections.abc import Awaitable, Callable, Iterable, Iterator +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator from dataclasses import dataclass from typing import Any @@ -90,6 +90,50 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True +@dataclass +class _Interrupted: + error: BaseException + + +async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: + try: + return await work + except (KeyboardInterrupt, SystemExit) as error: + # These exceptions otherwise stop asyncio's loop before the owner can close. + return _Interrupted(error) + + +async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: + """Bound owned work, reserving cleanup inside its original phase deadline. + + A failed borrowed transaction must remain alive until public close sends its + cancellation. Closing the transport first can leave its backend locks held. + """ + task = asyncio.create_task(_work_result(work)) + reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) + if not done: + raise TimeoutError("attach Postgres phase deadline expired") + result = task.result() + if isinstance(result, _Interrupted): + raise result.error + return result + except BaseException: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. + pg._terminate(conn) + raise + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -125,16 +169,11 @@ async def direct_refusals(conn: Any, what: str) -> list[Refusal]: outlive the CLI on the pooler's server connection, and attach's tenancy rows would be refused as an unchecked writer's.""" identities = [] - try: - for _ in range(2): - tx = conn.transaction() - await tx.start() - identities.append(await conn.fetchrow(_IDENTITY)) - await tx.commit() - except BaseException: - # A cancelled read must not wait for a transaction context's rollback. - conn.terminate() - raise + for _ in range(2): + tx = conn.transaction() + await tx.start() + identities.append(await conn.fetchrow(_IDENTITY)) + await tx.commit() first, second = identities out = [] if first is None or second is None or tuple(first) != tuple(second): @@ -538,30 +577,32 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S + return await supervise( + conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + ) + + +async def _trigger_try( + conn: Any, table: str, most_ms: int, wait: Wait, key: int +) -> cell_triggers.Outcome: + await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") try: - async with asyncio.timeout(bound_s): - await conn.execute(f"SET lock_timeout = '{wait.bound_ms(_MIGRATION_LOCK_MS)}ms'") - try: - await conn.execute(_MIGRATION_LOCK, _LOCK_KEY) - except Exception as e: - if getattr(e, "sqlstate", None) != "55P03": - raise - await conn.execute("RESET lock_timeout") - return cell_triggers.Outcome("busy", await _migration_holder(conn, _LOCK_KEY)) - await conn.execute("RESET lock_timeout") - outcome = await cell_triggers.ensure_table( - conn, - table, - recording_first=True, - lock_timeout=f"{wait.bound_ms(most_ms)}ms", - attempts=1, - ) - await conn.execute(_MIGRATION_UNLOCK, _LOCK_KEY) - return outcome - except BaseException: - # Closing the session releases its locks without asynchronous error cleanup. - conn.terminate() - raise + await conn.execute(_MIGRATION_LOCK, key) + except Exception as e: + if getattr(e, "sqlstate", None) != "55P03": + raise + await conn.execute("RESET lock_timeout") + return cell_triggers.Outcome("busy", await _migration_holder(conn, key)) + await conn.execute("RESET lock_timeout") + outcome = await cell_triggers.ensure_table( + conn, + table, + recording_first=True, + lock_timeout=f"{wait.bound_ms(most_ms)}ms", + attempts=1, + ) + await conn.execute(_MIGRATION_UNLOCK, key) + return outcome def _busy_refusal(table: str, holders: list[Any]) -> Refusal: diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 1899a46bab..68e74777ec 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -324,7 +324,8 @@ async def _run_ddl( two causes cells.md names kept it from applying (`busy`: its lock not taken, with the sessions it waited behind; `denied`: a table this role cannot alter), which a boot serves past; any other error fails it, as a failed read of the state does. `warn` logs - a busy table once, after its last try, with its DDL and holders (a boot's `ensure`).""" + a busy table once, after its last try, with its DDL and holders (a boot's `ensure`). + The borrowing caller owns failed-session cleanup, including cancellation/rollback.""" import asyncpg for attempt in range(attempts): @@ -344,7 +345,6 @@ async def _run_ddl( return Outcome("applied") except asyncpg.exceptions.InsufficientPrivilegeError as e: if not rolled_back: - conn.terminate() raise log.warning( "cell triggers: not applied on loom.%s (%s): this role cannot alter it", table, e @@ -352,7 +352,6 @@ async def _run_ddl( return Outcome("denied") except asyncpg.exceptions.LockNotAvailableError as e: if not rolled_back: - conn.terminate() raise # Short tries outlast a short transaction; an autovacuum is left to finish. if attempt < attempts - 1: @@ -368,11 +367,6 @@ async def _run_ddl( held, ) return Outcome("busy", held) - except BaseException: - if not rolled_back: - # Cancellation must not wait for another statement on this session. - conn.terminate() - raise return Outcome("busy") diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..145be1188d --- /dev/null +++ b/tests/integration/test_cell_attach_cleanup.py @@ -0,0 +1,111 @@ +"""Attach owners close active sessions before cancelling transaction work.""" + +import argparse +import asyncio +import contextlib +import time + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import cell_triggers, pg +from tests.integration.test_pg_owned_cleanup import database as database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.parametrize( + "owner,phase", + [ + ("trigger", "query"), + ("trigger", "rollback"), + ("mode", "query"), + ("mode", "rollback"), + ("direct", "query"), + ], +) +@pytest.mark.parametrize("interruption", ["cancel", "deadline", "second-cancel"]) +async def test_attach_owner_releases_backend_locks_before_return( + database, monkeypatch, owner, phase, interruption +): + import asyncpg + + conn, observer = database + await conn.execute( + "CREATE SCHEMA loom; CREATE TABLE loom.tenancy_mode(mode text, since timestamptz); INSERT INTO loom.tenancy_mode VALUES('recording', now())" + ) + pid = conn.get_server_pid() + execute = asyncpg.Connection.execute + closing = asyncio.Event() + close = asyncpg.Connection.close + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query.startswith("SET LOCAL statement_timeout"): + return await execute(self, "SET LOCAL statement_timeout = '10000ms'") + if query == "RAISE Python error": + raise ValueError("trigger statement failed") + if phase == "rollback" and query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + if owner == "mode" and query.startswith("LOCK TABLE"): + await execute(self, "SELECT * FROM probe FOR UPDATE") + if phase == "rollback": + raise ValueError("mode statement failed") + return await execute(self, "SELECT pg_sleep(10)") + if owner == "direct" and query.startswith("BEGIN"): + await execute(self, query) + return await execute(self, "SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + async def closing_session(self, *args, **kwargs): + was_open = not self.is_closed() + await close(self, *args, **kwargs) + if self is conn and was_open and interruption == "second-cancel": + closing.set() + await asyncio.Future() + + async def ensure(conn, table, **kwargs): + statements = ["SELECT * FROM probe FOR UPDATE"] + statements += ["RAISE Python error"] if phase == "rollback" else ["SELECT pg_sleep(10)"] + return await cell_triggers._run_ddl( + conn, table, statements, kwargs["lock_timeout"], 1, warn=False + ) + + @contextlib.asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + monkeypatch.setattr(asyncpg.Connection, "close", closing_session) + monkeypatch.setattr(cell_triggers, "ensure_table", ensure) + monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) + monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) + monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) + monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) + monkeypatch.setattr(pg, "dedicated", dedicated) + if owner == "trigger": + work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + elif owner == "mode": + work = apg._mode_try(conn, 100, lambda _: None) + else: + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + work = run.open_lock() + task = asyncio.create_task(work) + try: + await sleeping(observer, pid) + started = time.monotonic() + if interruption != "deadline": + task.cancel() + if interruption == "second-cancel": + await asyncio.wait_for(closing.wait(), 1) + task.cancel() + with pytest.raises(TimeoutError if interruption == "deadline" else asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 1.2) + assert time.monotonic() - started < 0.7 + await released(observer, pid) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 65feba889d..0b3401be88 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,6 +8,7 @@ import argparse import asyncio import json +import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -524,6 +525,24 @@ async def close(self, *, timeout: float) -> None: def terminate(self) -> None: self.executed.append("TERMINATE") + async def close(self, **kwargs: Any) -> None: + assert kwargs["timeout"] > 0 + self.executed.append("CLOSE") + + def is_closed(self) -> bool: + return bool(self.executed and self.executed[-1] in {"CLOSE", "TERMINATE"}) + + +def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + supervise = apg.supervise + + async def bounded(conn: Any, work: Any, *, end: float) -> Any: + if bounds is not None: + bounds.append(end - time.monotonic()) + return await supervise(conn, work, end=time.monotonic() + 0.01) + + monkeypatch.setattr(apg, "supervise", bounded) + def _seq(*values: Any) -> Callable[..., Any]: pending = list(values) @@ -770,13 +789,7 @@ async def execute(self, sql: str, *args: Any) -> str: conn = _StalledCommit([("UPDATE loom.tenancy_mode", "UPDATE 1")]) bounds: list[float] = [] - timeout = asyncio.timeout - - def bounded(seconds: float) -> Any: - bounds.append(seconds) - return timeout(0) - - monkeypatch.setattr(apg.asyncio, "timeout", bounded) + _short_phases(monkeypatch, bounds) marks: list[str] = [] async def confirm() -> None: @@ -800,8 +813,9 @@ async def ready() -> list[str]: timeout=0.2, ) assert "COMMIT" in conn.executed - assert conn.executed[-1] == "TERMINATE" - assert bounds == [bound_s] + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed + assert bounds == pytest.approx([bound_s], abs=0.001) assert marks == ["unknown"] @@ -820,8 +834,7 @@ async def execute(self, sql: str, *args: Any) -> str: return result conn = _StalledRollback() - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task( apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) @@ -833,7 +846,8 @@ async def execute(self, sql: str, *args: Any) -> str: await task assert task in done, "mode deadline is stuck awaiting rollback" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == (["unknown"] if stall == "UPDATE" else []) @@ -875,7 +889,8 @@ async def execute(self, sql: str, *args: Any) -> str: assert task.cancelled() else: assert isinstance(task.exception(), ValueError) - assert conn.executed[-1] == "TERMINATE" + assert "CLOSE" not in conn.executed + assert "TERMINATE" not in conn.executed assert "ROLLBACK" not in conn.executed @@ -901,7 +916,8 @@ async def ready() -> list[str]: expected_cell="first", expected_fp=(CURRENT, NEXT), ) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed assert marks == [] @@ -1106,8 +1122,7 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: conn, table, ["CREATE TRIGGER stub"], kwargs["lock_timeout"], 1, warn=False ) - timeout = asyncio.timeout - monkeypatch.setattr(apg.asyncio, "timeout", lambda _s: timeout(0)) + _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) @@ -1115,7 +1130,8 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: try: assert task in done, f"trigger deadline is stuck at {stall}" assert isinstance(task.exception(), TimeoutError) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed if stall in {"migration", "ddl", "commit", "rollback"}: assert apg._MIGRATION_UNLOCK not in conn.executed finally: @@ -1147,7 +1163,8 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) - assert conn.executed[-1] == "TERMINATE" + assert conn.executed[-1] == "CLOSE" + assert "TERMINATE" not in conn.executed @pytest.mark.asyncio diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py new file mode 100644 index 0000000000..20e17ac786 --- /dev/null +++ b/tests/unit/test_cell_attach_cleanup.py @@ -0,0 +1,119 @@ +"""The attach owner preserves failures through bounded public-close cleanup.""" + +import argparse +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_attach +from server.cli import cell_attach_pg as apg +from server.core import pg + + +@pytest.mark.asyncio +async def test_failed_work_keeps_its_error_when_public_close_also_fails(): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + raise RuntimeError("close failed") + + def terminate(self): + events.append("abort") + + async def work(): + raise ValueError("original work failure") + + with pytest.raises(ValueError, match="original work failure"): + await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) + assert events == ["close", "abort"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_second_cancellation_preserves_original_failure_and_closes_before_cancel(failed): + entered, closing = asyncio.Event(), asyncio.Event() + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("close") + closing.set() + await asyncio.Future() + + def terminate(self): + events.append("abort") + + async def work(): + entered.set() + try: + if failed: + raise ValueError("original work failure") + await asyncio.Future() + finally: + events.append("work ended") + + task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + await entered.wait() + if not failed: + task.cancel("first cancellation") + await closing.wait() + task.cancel("second cancellation") + with pytest.raises( + ValueError if failed else asyncio.CancelledError, + match="original work failure" if failed else "first cancellation", + ): + await task + assert events.count("abort") >= 1 + if not failed: + assert events.index("close") < events.index("work ended") + + +@pytest.mark.asyncio +async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + cm = AsyncMock() + exit_started = asyncio.Event() + + async def exit(*_): + exit_started.set() + await asyncio.Future() + + cm.__aexit__.side_effect = exit + run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) + run._lock_cm, run.lock_conn = cm, conn + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + started = time.monotonic() + with pytest.raises(TimeoutError): + await run.close_lock() + assert time.monotonic() - started < 0.15 + assert exit_started.is_set() and run.lock_conn is None + conn.close.assert_awaited_once() + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", [False, True]) +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): + router = AsyncMock() + router.call.return_value = {"state": "attaching"} + run = cell_attach.Run( + None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None + ) + run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.dry_run = AsyncMock(return_value=0) + run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) + try: + raise LookupError("unrelated caller exception") + except LookupError: + with pytest.raises( + ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" + ): + await run.run() From 946ef2f0b09b2ef34aa0cb3633eee8ce66e5f9a4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:24:29 +0000 Subject: [PATCH 047/181] Validate prospective cell settings without changing process configuration Signed-off-by: zlareb1 --- server/cli/cell_update_config.py | 45 ++++++ server/code/domain/index/repos.py | 73 +++++---- server/core/cell.py | 35 +++-- tests/unit/test_cell.py | 2 +- tests/unit/test_cell_update_config.py | 138 ++++++++++++++++++ .../unit/test_code_repo_entry_clone_fields.py | 8 +- tests/unit/test_code_scip_extra_producers.py | 2 +- tests/unit/test_repo_snapshot_contract.py | 2 +- 8 files changed, 251 insertions(+), 54 deletions(-) create mode 100644 server/cli/cell_update_config.py create mode 100644 tests/unit/test_cell_update_config.py diff --git a/server/cli/cell_update_config.py b/server/cli/cell_update_config.py new file mode 100644 index 0000000000..26ee346e7f --- /dev/null +++ b/server/cli/cell_update_config.py @@ -0,0 +1,45 @@ +"""An explicit nonsecret settings overlay for a cell's target configuration.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import ValidationError +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource + +from server.settings import Settings +from server.settings._file import SettingsFileError, read_settings_file +from server.settings._live import coerce + + +class _Target(Settings): + """Validate the supplied complete model without rereading external sources.""" + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return (init_settings,) + + +def read_target(path: Path, current: Settings) -> Settings: + """Overlay FILE on the CLI's effective settings; install neither the file nor values.""" + values = read_settings_file(str(path), set(Settings.model_fields)) + checked: dict[str, Any] = {} + for name, value in values.items(): + try: + checked[name] = coerce(Settings, name, value) + except ValidationError as error: + rule = error.errors(include_input=False)[0]["msg"] + raise SettingsFileError(f"{path}: {name} has an invalid value ({rule})") from None + try: + return _Target(**(current.model_dump() | checked)) + except ValidationError as error: + # Settings hides inputs, including a model-level validator's complete input. + raise SettingsFileError(f"{path}: invalid target configuration ({error})") from None diff --git a/server/code/domain/index/repos.py b/server/code/domain/index/repos.py index beb29b8819..b382e648e7 100644 --- a/server/code/domain/index/repos.py +++ b/server/code/domain/index/repos.py @@ -41,7 +41,7 @@ from typing import Any from server.core.errors import BackendUnavailableError, ValidationError -from server.settings import settings +from server.settings import Settings, settings logger = logging.getLogger("loom.code_index.repos") @@ -179,27 +179,27 @@ def derived_git_dir(repo: str, root: str = CLONE_ROOT, *, fallback: str = "") -> return os.path.join(root, f"{name}.git") if name else "" -def _scalar_repo() -> RepoConfig: +def _scalar_repo(config: Settings) -> RepoConfig: """The scalar settings as one entry. The git dir and remote, when unset, are derived from `code_repo`; with no repository they stay empty (`configured_repos()` then lists no scalar entry).""" - repo = str(settings.code_repo).strip() + repo = str(config.code_repo).strip() return RepoConfig( - org=settings.code_org, - namespace=settings.code_namespace, + org=config.code_org, + namespace=config.code_namespace, repo=repo, - git_dir=settings.code_git_dir or derived_git_dir(repo), - git_remote=settings.code_git_remote or derived_git_remote(repo), - git_branch=settings.code_git_branch, - path_prefixes=_prefixes(settings.code_path_prefixes) or WHOLE_TREE, - github_token=settings.code_github_token, - org_readable=bool(settings.code_org_readable), - public=bool(settings.code_repo_public), - scip_checkout_dir=settings.code_scip_checkout_dir, - git_extra_branches=str(settings.code_git_extra_branches), - git_fetch_pr_heads=bool(settings.code_git_fetch_pr_heads), - exclude=_prefixes(settings.code_ingest_exclude), - ignore_file=str(settings.code_ingest_ignore_file), + git_dir=config.code_git_dir or derived_git_dir(repo), + git_remote=config.code_git_remote or derived_git_remote(repo), + git_branch=config.code_git_branch, + path_prefixes=_prefixes(config.code_path_prefixes) or WHOLE_TREE, + github_token=config.code_github_token, + org_readable=bool(config.code_org_readable), + public=bool(config.code_repo_public), + scip_checkout_dir=config.code_scip_checkout_dir, + git_extra_branches=str(config.code_git_extra_branches), + git_fetch_pr_heads=bool(config.code_git_fetch_pr_heads), + exclude=_prefixes(config.code_ingest_exclude), + ignore_file=str(config.code_ingest_ignore_file), ) @@ -208,7 +208,7 @@ def default_git_dir() -> str: no scalar repository is configured or `configured_repos()` does not list it (a namespace added from the page skips it). The fallback of a read whose namespace has no entry.""" - base = _scalar_repo() + base = _scalar_repo(settings) listed = ( c for c in configured_repos() @@ -360,7 +360,7 @@ def configured_repos() -> list[RepoConfig]: degrades to the scalar single-repo config with a loud log (a typo must not silently stop all indexing) — loud once, then a gauge.""" require_registry_read() - return _with_registry(_configured_from_env()) + return _with_registry(_operator_entries()) def require_registry_read() -> None: @@ -392,7 +392,12 @@ def operator_orgs() -> set[str]: """The orgs the operator's entries name (`LOOM_CODE_REPOS_JSON`, else the scalar repository), without the self-service registry's: a cell's configured orgs (`server.core.cell`).""" - return {c.org for c in _configured_from_env() if c.org} + return {c.org for c in _operator_entries() if c.org} + + +def operator_orgs_for(config: Settings) -> set[str]: + """Configured orgs without changing registry state or the running diagnostics.""" + return {c.org for c in _configured_from_env(config, {}) if c.org} def config_entries() -> list[RepoConfig]: @@ -403,7 +408,7 @@ def config_entries() -> list[RepoConfig]: restricted through an access row reads as its configuration says until then. Never for reading, fetching or indexing a repository: `tests/unit/test_repo_snapshot_contract.py` pins its callers.""" - return _with_registry(_configured_from_env()) if registry_unread() else configured_repos() + return _with_registry(_operator_entries()) if registry_unread() else configured_repos() def config_entry(org: str, namespace: str) -> RepoConfig | None: @@ -461,10 +466,10 @@ def _with_registry(env_repos: list[RepoConfig]) -> list[RepoConfig]: return out + extra -def _scalar_entries(errors: dict[str, str]) -> list[RepoConfig]: +def _scalar_entries(config: Settings, errors: dict[str, str]) -> list[RepoConfig]: """The scalar entry, or none: an empty `code_repo` configures no repository, and a repository without its org or namespace is a configuration error, not an entry.""" - base = _scalar_repo() + base = _scalar_repo(config) if not base.repo: return [] missing = [ @@ -481,12 +486,18 @@ def _scalar_entries(errors: dict[str, str]) -> list[RepoConfig]: return [base] -def _configured_from_env() -> list[RepoConfig]: - raw = settings.code_repos_json.strip() +def _operator_entries() -> list[RepoConfig]: errors: dict[str, str] = {} + result = _configured_from_env(settings, errors) + _publish_config_errors(errors) + return result + + +def _configured_from_env(config: Settings, errors: dict[str, str]) -> list[RepoConfig]: + """Parse explicit operator settings, collecting diagnostics without publishing them.""" + raw = config.code_repos_json.strip() if not raw: - scalar = _scalar_entries(errors) - _publish_config_errors(errors) + scalar = _scalar_entries(config, errors) return scalar try: entries = json.loads(raw) @@ -497,10 +508,9 @@ def _configured_from_env() -> list[RepoConfig]: "LOOM_CODE_REPOS_JSON is not a JSON array — falling back to the " "scalar single-repo config" ) - scalar = _scalar_entries(errors) - _publish_config_errors(errors) + scalar = _scalar_entries(config, errors) return scalar - base = _scalar_repo() + base = _scalar_repo(config) repos: list[RepoConfig] = [] seen: set[str] = set() for i, e in enumerate(entries): @@ -587,8 +597,7 @@ def _configured_from_env() -> list[RepoConfig]: ) ) if not repos: - repos = _scalar_entries(errors) - _publish_config_errors(errors) + repos = _scalar_entries(config, errors) return repos diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..b1eb2a0b47 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -46,7 +46,7 @@ from server.core import pg, readiness from server.core.errors import caller_text -from server.settings import settings +from server.settings import Settings, settings from server.settings.catalog import is_http_url log = logging.getLogger(__name__) @@ -180,33 +180,38 @@ def _code_repo_orgs() -> set[str]: def fingerprint(deployment_id: str) -> dict[str, Any]: """This process's fingerprint: what the cell accepts, which the directory checks against every other cell, and the stores it serves from.""" - public = settings.public_base_url - end_users = settings.end_user_issuers() - bucket = settings.document_s3_bucket + return fingerprint_for(settings, deployment_id, _code_repo_orgs()) + + +def fingerprint_for(config: Settings, deployment_id: str, code_orgs: set[str]) -> dict[str, Any]: + """The same fingerprint for an explicit, validated configuration.""" + public = config.public_base_url + end_users = config.end_user_issuers() + bucket = config.document_s3_bucket configured = { o for o in ( - settings.oidc_jit_org, - settings.self_signup_org if settings.self_signup == "open" else "", - settings.code_org, - settings.tracker_org, - *_code_repo_orgs(), + config.oidc_jit_org, + config.self_signup_org if config.self_signup == "open" else "", + config.code_org, + config.tracker_org, + *code_orgs, ) if o } return { "public_host": normalise_host(public), - "oauth_issuer_host": normalise_host(settings.oauth_issuer or public), - "oidc_issuer": settings.oidc_issuer.rstrip("/"), - "oidc_client_id": settings.oidc_client_id, - "scim": bool(settings.scim_token.strip()), - "bearer_issuers": sorted(settings.trusted_issuers() | set(end_users)), + "oauth_issuer_host": normalise_host(config.oauth_issuer or public), + "oidc_issuer": config.oidc_issuer.rstrip("/"), + "oidc_client_id": config.oidc_client_id, + "scim": bool(config.scim_token.strip()), + "bearer_issuers": sorted(config.trusted_issuers() | set(end_users)), "configured_orgs": sorted(configured), "end_user_orgs": sorted({o for spec in end_users.values() for o in spec.orgs}), "deployment_id": deployment_id, "bucket": bucket, # Without a bucket the prefix stores nothing and takes part in no check. - "bucket_prefix": settings.document_s3_prefix if bucket else "", + "bucket_prefix": config.document_s3_prefix if bucket else "", } diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index dcd2536ac4..d35f2206a1 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -760,7 +760,7 @@ def test_the_configured_orgs_hold_the_org_of_each_repository_entry_the_code_conf monkeypatch.setattr(settings, "code_repo", code_repo) monkeypatch.setattr(settings, "code_repos_json", repos_json) monkeypatch.setattr(cell, "_code_orgs", repos.operator_orgs) - kept = {r.org for r in repos._configured_from_env()} + kept = {r.org for r in repos._configured_from_env(settings, {})} assert set(cell.fingerprint("dep-1")["configured_orgs"]) == ({code_org} | kept) - {""} diff --git a/tests/unit/test_cell_update_config.py b/tests/unit/test_cell_update_config.py new file mode 100644 index 0000000000..fd1721ec42 --- /dev/null +++ b/tests/unit/test_cell_update_config.py @@ -0,0 +1,138 @@ +"""An update calculates its target without changing the running configuration.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from server.code.domain.index import repos +from server.core import cell +from server.settings import Settings, settings +from server.settings._file import SettingsFileError + + +def _config(**values: object) -> Settings: + return Settings(_env_file=None, **(settings.model_dump() | values)) + + +def test_the_overlay_wins_over_old_environment_without_installing_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from server.cli.cell_update_config import read_target + + current = _config(code_org="before", tracker_org="tracker") + monkeypatch.setenv("LOOM_CODE_ORG", "environment") + # A candidate validates supplied values, without rereading another settings file. + monkeypatch.setenv("LOOM_SETTINGS_FILE", str(tmp_path / "absent.yaml")) + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n code_org: after\n") + before_env, before_settings = dict(os.environ), settings.model_dump() + target = read_target(path, current) + assert target.code_org == "after" + assert target.tracker_org == "tracker" + assert current.code_org == "before" + assert dict(os.environ) == before_env + assert settings.model_dump() == before_settings + + +@pytest.mark.parametrize( + "body,field", + [ + (" code_org: one\n code_org: two\n", "code_org"), + (" jwt_secret: do-not-copy-this-value\n", "jwt_secret"), + (" memory_ask_deep_evidence_max_memories: -1\n", "memory_ask_deep_evidence_max_memories"), + ], +) +def test_invalid_overlay_fields_refuse_without_echoing_values( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, body: str, field: str +) -> None: + from server.cli.cell_update_config import read_target + + current = _config() + monkeypatch.setenv("LOOM_MEMORY_ASK_DEEP_EVIDENCE_MAX_MEMORIES", "5") + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n" + body) + with pytest.raises(SettingsFileError) as exc: + read_target(path, current) + assert field in str(exc.value) + assert "do-not-copy-this-value" not in str(exc.value) + + +def test_full_model_rules_apply_and_unknown_settings_keep_the_normal_warning( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + from server.cli.cell_update_config import read_target + + current = _config() + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings:\n future_cell_field: ignored\n tracker_org: next\n") + assert read_target(path, current).tracker_org == "next" + assert "future_cell_field" in caplog.text + path.write_text( + "version: 1\nsettings:\n inference_token_audience: own-audience\n public_base_url: ''\n" + ) + with pytest.raises(SettingsFileError, match="LOOM_INFERENCE_TOKEN_URL"): + read_target(path, current) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("", {"scalar"}), + ("not json", {"scalar"}), + ("[]", {"scalar"}), + ('[{"namespace":"code-only"}]', {"scalar"}), + ( + '[{"namespace":"first","org":"one","repo":"a/b"},' + '{"namespace":"first","org":"duplicate"},' + '{"namespace":"bad.name","org":"skipped"},' + '{"namespace":"second","org":"two"}]', + {"one", "two"}, + ), + ], +) +def test_explicit_code_parser_keeps_process_semantics_without_registry_or_gauge_changes( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: set[str] +) -> None: + # The parser retains its existing defensive fallback for a corrupt runtime value. + config = _config(code_org="scalar", code_repo="a/b", code_namespace="code").model_copy( + update={"code_repos_json": raw} + ) + for key in ("code_org", "code_repo", "code_namespace", "code_repos_json"): + monkeypatch.setattr(settings, key, getattr(config, key)) + monkeypatch.setattr(repos, "_config_errors", {}) + monkeypatch.setattr(repos, "_config_errors_logged", set()) + monkeypatch.setattr(repos, "_config_evaluated", False) + process_result = repos.operator_orgs() + errors = dict(repos._config_errors) + logged = set(repos._config_errors_logged) + + def no_registry() -> object: + raise AssertionError("candidate read a member-managed registry") + + monkeypatch.setattr(repos, "registry_repos", no_registry) + assert repos.operator_orgs_for(config) == process_result == expected + assert repos._config_errors == errors + assert repos._config_errors_logged == logged + + +def test_explicit_fingerprint_is_identical_to_the_process_and_uses_candidate_orgs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = _config(code_org="before", tracker_org="tracker") + for name in Settings.model_fields: + monkeypatch.setattr(settings, name, getattr(current, name)) + monkeypatch.setattr(cell, "_code_orgs", lambda: {"extra"}) + fields = cell.fingerprint_for(current, "deployment", {"extra"}) + assert json.dumps(fields, sort_keys=True) == json.dumps( + cell.fingerprint("deployment"), sort_keys=True + ) + target = _config(code_org="after", tracker_org="tracker") + changed = cell.fingerprint_for(target, "deployment", {"target-repo"}) + assert "after" in changed["configured_orgs"] + assert "target-repo" in changed["configured_orgs"] + assert "before" not in changed["configured_orgs"] + assert cell.fingerprint("deployment") == fields diff --git a/tests/unit/test_code_repo_entry_clone_fields.py b/tests/unit/test_code_repo_entry_clone_fields.py index b03e462065..795028d8e7 100644 --- a/tests/unit/test_code_repo_entry_clone_fields.py +++ b/tests/unit/test_code_repo_entry_clone_fields.py @@ -46,7 +46,7 @@ def test_release_branch_of_the_scalar_repo_still_inherits_its_clone( {"namespace": "code-clickhouse-26-1", "git_branch": "26.1"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env()} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, {})} rel = by_ns["code-clickhouse-26-1"] assert rel.git_dir == "var/code_repos/clickhouse.git" assert rel.git_remote == "https://github.com/ClickHouse/ClickHouse.git" @@ -65,7 +65,7 @@ def test_a_different_repo_derives_its_remote_from_repo(monkeypatch: pytest.Monke } ], ) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, {}) assert cfg.git_remote == "https://github.com/langfuse/langfuse.git" assert cfg.git_dir == "var/code_repos/langfuse.git" assert not captured @@ -83,7 +83,7 @@ def test_an_explicit_remote_is_kept(monkeypatch: pytest.MonkeyPatch) -> None: } ], ) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, {}) assert cfg.git_remote == "git@github.com:acme/x.git" @@ -97,7 +97,7 @@ def test_a_different_repo_without_git_dir_gets_a_clone_of_its_own( {"namespace": "code-langfuse", "repo": "langfuse/langfuse"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env()} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, {})} lf = by_ns["code-langfuse"] assert lf.git_dir == "var/code_repos/langfuse.git" assert lf.git_dir != by_ns["code-clickhouse"].git_dir diff --git a/tests/unit/test_code_scip_extra_producers.py b/tests/unit/test_code_scip_extra_producers.py index 4a157acc96..dc3edaa2a8 100644 --- a/tests/unit/test_code_scip_extra_producers.py +++ b/tests/unit/test_code_scip_extra_producers.py @@ -68,7 +68,7 @@ def test_the_entry_json_carries_the_extra_producers_and_the_projects(monkeypatch "scip_ts_projects": "frontend, sdk/typescript", } monkeypatch.setattr(settings, "code_repos_json", json.dumps([entry])) - (cfg,) = repos._configured_from_env() + (cfg,) = repos._configured_from_env(settings, {}) assert cfg.scip_extra_producers == ("typescript",) assert cfg.scip_ts_projects == ("frontend", "sdk/typescript") diff --git a/tests/unit/test_repo_snapshot_contract.py b/tests/unit/test_repo_snapshot_contract.py index afd1ca2f12..bc46346a81 100644 --- a/tests/unit/test_repo_snapshot_contract.py +++ b/tests/unit/test_repo_snapshot_contract.py @@ -158,7 +158,7 @@ def test_the_accessors_refuse_until_the_registry_is_read(monkeypatch) -> None: org_readable=True, ) monkeypatch.setattr(settings, "code_self_service_enabled", True) - monkeypatch.setattr(repos, "_configured_from_env", lambda: [entry]) + monkeypatch.setattr(repos, "_configured_from_env", lambda _config, _errors: [entry]) monkeypatch.setattr(repos, "registry_repos", lambda: []) monkeypatch.setattr(repos, "_registry_loaded", False) with pytest.raises(repos.RegistryUnread): From c9d1f0a514e84942714844f28f631755822cda34 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:39:15 +0000 Subject: [PATCH 048/181] Check cell fingerprint reservations with shared org pattern rules Signed-off-by: zlareb1 --- server/core/org_patterns.py | 19 +++ server/router/directory.py | 25 +--- server/router/fingerprints.py | 114 +++++++++++++++++ server/settings/catalog.py | 18 +-- .../test_cell_fingerprint_reservations.py | 121 ++++++++++++++++++ 5 files changed, 261 insertions(+), 36 deletions(-) create mode 100644 server/core/org_patterns.py create mode 100644 server/router/fingerprints.py create mode 100644 tests/unit/test_cell_fingerprint_reservations.py diff --git a/server/core/org_patterns.py b/server/core/org_patterns.py new file mode 100644 index 0000000000..0b7d89551d --- /dev/null +++ b/server/core/org_patterns.py @@ -0,0 +1,19 @@ +"""The exact-name and trailing-star org patterns used by end-user issuers.""" + +from __future__ import annotations + + +def matches(pattern: str, org: str) -> bool: + """Match one literal org; only the pattern's trailing star has special meaning.""" + return org.startswith(pattern[:-1]) if pattern.endswith("*") else org == pattern + + +def overlap(a: str, b: str) -> bool: + """Whether two exact names or prefix patterns can match the same org.""" + if a.endswith("*") and b.endswith("*"): + return a[:-1].startswith(b[:-1]) or b[:-1].startswith(a[:-1]) + if a.endswith("*"): + return matches(a, b) + if b.endswith("*"): + return matches(b, a) + return a == b diff --git a/server/router/directory.py b/server/router/directory.py index a92967ed86..235c97fa66 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -28,7 +28,7 @@ from server.core import cell as cell_state from server.core.paths import ROUTER_MIGRATIONS_DIR -from server.router.limits import MAX_BODY_BYTES +from server.router import fingerprints from server.settings import settings # The directory's own migration lock: never the memory authority's. @@ -805,27 +805,6 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> return "" -# An org as a placement body carries it at most: ORG_MAX_BYTES bytes, each escaped in JSON -# to at most six (`\u00XX`). -_PLACEMENT_ORG_MAX = 6 * cell_state.ORG_MAX_BYTES # not a setting: derived - - -def _placement_refusal(cell: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" - # The default encoding is the longest: spaces after separators, non-ASCII escaped. - size = len(json.dumps({"org": "", "cell": cell, "fingerprint": value})) + _PLACEMENT_ORG_MAX - if size <= MAX_BODY_BYTES: - return "" - sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) - largest = ", ".join(k for _, k in sizes[:3]) - return ( - f"{name} makes a placement body of up to {size} bytes, past the router's " - f"{MAX_BODY_BYTES}: its largest fields are {largest}" - ) - - def _as_json(*values: Any) -> str: """Values in one canonical JSON text, which tells `false`, `0` and `0.0` apart.""" return json.dumps(values, sort_keys=True) @@ -840,7 +819,7 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: for name, value in (("current_fp", begin.current_fp), ("next_fp", begin.next_fp)): if value is None: continue - if reason := _fingerprint_refusal(found, name, value) or _placement_refusal( + if reason := _fingerprint_refusal(found, name, value) or fingerprints.placement_refusal( row.cell, name, value ): return reason diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py new file mode 100644 index 0000000000..5f8778b3a6 --- /dev/null +++ b/server/router/fingerprints.py @@ -0,0 +1,114 @@ +"""Pure checks on the fingerprint reservations shared by cell controllers.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + +from server.core import cell, org_patterns +from server.router.limits import MAX_BODY_BYTES + + +def readable(value: Any) -> bool: + """Known fields have exact types; unknown JSON fields remain reserved and intact.""" + if not isinstance(value, dict) or set(value) != {"v", "fields"}: + return False + if type(value["v"]) is not int or value["v"] != cell.FINGERPRINT_VERSION: + return False + fields = value["fields"] + if not isinstance(fields, dict): + return False + try: + # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + return False + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + try: + if "\0" in item: + return False + item.encode() + except UnicodeError: + return False + elif isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + return False + pending.extend(item.keys()) + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + elif item is not None and type(item) not in (bool, int, float): + return False + for name, default in cell.DEFAULTS.items(): + item = fields.get(name, default) + if type(item) is not type(default): + return False + if isinstance(item, list) and any(not isinstance(part, str) for part in item): + return False + for name in ("configured_orgs", "end_user_orgs"): + for org in fields.get(name, []): + if cell.named_org(org) is None or cell.org_name_refusal(org): + return False + return True + + +def _pair_refusal(a: dict[str, Any], b: dict[str, Any]) -> str: + if set(a.get("bearer_issuers", [])) & set(b.get("bearer_issuers", [])): + return "shared bearer issuer reservation" + oidc_a = (a.get("oidc_issuer", "").rstrip("/"), a.get("oidc_client_id", "")) + oidc_b = (b.get("oidc_issuer", "").rstrip("/"), b.get("oidc_client_id", "")) + if all(oidc_a) and oidc_a == oidc_b: + return "shared OIDC issuer and client reservation" + patterns_a, patterns_b = a.get("end_user_orgs", []), b.get("end_user_orgs", []) + if any(org_patterns.overlap(x, y) for x in patterns_a for y in patterns_b): + return "overlapping end-user org pattern reservation" + orgs_a, orgs_b = a.get("configured_orgs", []), b.get("configured_orgs", []) + if ( + set(orgs_a) & set(orgs_b) + or any(org_patterns.matches(pattern, org) for pattern in patterns_a for org in orgs_b) + or any(org_patterns.matches(pattern, org) for pattern in patterns_b for org in orgs_a) + ): + return "overlapping configured org reservation" + return "" + + +def reservation_refusal(accepted: Sequence[dict[str, Any]], other: Sequence[dict[str, Any]]) -> str: + """Compare every accepted value, including next, with another cell's reservation. + + Registration identity, placement winners and transaction locks are the caller's + checks. Empty `other` means registration has not reserved a fingerprint yet. + """ + if not accepted or any(not readable(value) for value in (*accepted, *other)): + return "unreadable fingerprint reservation" + for a in accepted: + for b in other: + if reason := _pair_refusal(a["fields"], b["fields"]): + return reason + return "" + + +# An org as a placement body carries it at most: ORG_MAX_BYTES bytes, each escaped in JSON +# to at most six (`\u00XX`). +_PLACEMENT_ORG_MAX = 6 * cell.ORG_MAX_BYTES # not a setting: derived + + +def placement_refusal(cell_name: str, name: str, value: dict[str, Any]) -> str: + """Why a fingerprint is too large for the cell's live placements ('' when it is not): a + placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's + MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" + # The default encoding is the longest: spaces after separators, non-ASCII escaped. + size = ( + len(json.dumps({"org": "", "cell": cell_name, "fingerprint": value})) + _PLACEMENT_ORG_MAX + ) + if size <= MAX_BODY_BYTES: + return "" + sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) + largest = ", ".join(k for _, k in sizes[:3]) + return ( + f"{name} makes a placement body of up to {size} bytes, past the router's " + f"{MAX_BODY_BYTES}: its largest fields are {largest}" + ) diff --git a/server/settings/catalog.py b/server/settings/catalog.py index b98c41fbbb..c967b5cf0b 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -28,6 +28,7 @@ import idna from pydantic import AfterValidator, BaseModel, Field, PrivateAttr, field_validator, model_validator +from server.core import org_patterns from server.core.document_formats import route_key from server.core.embedding_space import inferred_provider from server.core.numbers import parse_budget_value @@ -97,17 +98,6 @@ def _public_numbers(pem: str) -> object: return None -def _overlap(a: str, b: str) -> bool: - """Whether two org patterns (a name, or `prefix*`) can match one org.""" - if a.endswith("*") and b.endswith("*"): - return a[:-1].startswith(b[:-1]) or b[:-1].startswith(a[:-1]) - if a.endswith("*"): - return b.startswith(a[:-1]) - if b.endswith("*"): - return a.startswith(b[:-1]) - return a == b - - @dataclass(frozen=True) class EndUserIssuer: """One end-user issuer: the orgs its tokens may name and where its keys are.""" @@ -148,7 +138,7 @@ def parse(cls, issuer: str, spec: object) -> EndUserIssuer: def signs_for(self, org: str) -> bool: """Whether this issuer's tokens may name `org`: listed, or under a `prefix*`.""" - return any(org.startswith(o[:-1]) if o.endswith("*") else org == o for o in self.orgs) + return any(org_patterns.matches(o, org) for o in self.orgs) class AuthSettings(BaseModel): @@ -515,7 +505,9 @@ def _end_user_issuers_stand_apart(self) -> AuthSettings: raise ValueError(f"end-user issuer {issuer!r}: keys {same} are federated keys") # `user:` names one person per org, so one issuer alone signs for an org. for (a, spec_a), (b, spec_b) in itertools.combinations(issuers.items(), 2): - shared = [(x, y) for x in spec_a.orgs for y in spec_b.orgs if _overlap(x, y)] + shared = [ + (x, y) for x in spec_a.orgs for y in spec_b.orgs if org_patterns.overlap(x, y) + ] if shared: raise ValueError(f"end-user issuers {a!r} and {b!r} both sign for {shared[0]}") return self diff --git a/tests/unit/test_cell_fingerprint_reservations.py b/tests/unit/test_cell_fingerprint_reservations.py new file mode 100644 index 0000000000..37926b479b --- /dev/null +++ b/tests/unit/test_cell_fingerprint_reservations.py @@ -0,0 +1,121 @@ +"""A cell reserves both accepted values against every other cell's values.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell + + +def _value(**fields: Any) -> dict[str, Any]: + return cell.as_value(deepcopy(cell.DEFAULTS) | fields) + + +@pytest.mark.parametrize( + "left,right,reason", + [ + ({"bearer_issuers": ["issuer"]}, {"bearer_issuers": ["issuer"]}, "bearer"), + ( + {"oidc_issuer": "https://login/", "oidc_client_id": "client"}, + {"oidc_issuer": "https://login", "oidc_client_id": "client"}, + "OIDC", + ), + ({"end_user_orgs": ["team-*"]}, {"end_user_orgs": ["team-one*"]}, "pattern"), + ({"end_user_orgs": ["team-*"]}, {"configured_orgs": ["team-one"]}, "configured"), + ({"configured_orgs": ["team-one"]}, {"end_user_orgs": ["team-*"]}, "configured"), + ({"configured_orgs": ["same"]}, {"configured_orgs": ["same"]}, "configured"), + ], +) +def test_conflicts_include_next_and_both_directions( + left: dict[str, Any], right: dict[str, Any], reason: str +) -> None: + from server.router.fingerprints import reservation_refusal + + assert reason in reservation_refusal([_value(), _value(**left)], [_value(**right)]) + assert reason in reservation_refusal([_value(**left)], [_value(), _value(**right)]) + assert reason in reservation_refusal([_value(**right)], [_value(**left)]) + + +def test_configured_orgs_are_literal_and_empty_oidc_pair_is_unreserved() -> None: + from server.router.fingerprints import reservation_refusal + + assert ( + reservation_refusal( + [_value(configured_orgs=["team*"])], [_value(configured_orgs=["teams"])] + ) + == "" + ) + assert ( + reservation_refusal([_value(configured_orgs=["team*"])], [_value(end_user_orgs=["teams*"])]) + == "" + ) + assert reservation_refusal([_value(oidc_issuer="issuer")], [_value(oidc_issuer="issuer")]) == "" + assert ( + reservation_refusal([_value(oidc_client_id="client")], [_value(oidc_client_id="client")]) + == "" + ) + + +@pytest.mark.parametrize( + "value", + [ + {"v": True, "fields": {}}, + {"v": 99, "fields": {}}, + {"v": 1, "fields": {"scim": 0}}, + {"v": 1, "fields": {"bearer_issuers": "issuer"}}, + {"v": 1, "fields": {"configured_orgs": ["*"]}}, + {"v": 1, "fields": {"configured_orgs": ["-"]}}, + {"v": 1, "fields": {"configured_orgs": [""]}}, + {"v": 1, "fields": {"configured_orgs": ["x" * (cell.ORG_MAX_BYTES + 1)]}}, + {"v": 1, "fields": {"end_user_orgs": ["*"]}}, + {"v": 1, "fields": {"public_host": "bad\0host"}}, + {"v": 1, "fields": {"future": {"nested": "\ud800"}}}, + {"v": 1, "fields": {"future": float("nan")}}, + {"v": 1, "fields": {"future": ("hidden\0text",)}}, + {"v": 1, "fields": {"future": {1: "non-string key"}}}, + ], +) +def test_unreadable_reservations_refuse_on_either_side(value: Any) -> None: + from server.router.fingerprints import reservation_refusal + + assert "unreadable" in reservation_refusal([_value()], [value]) + assert "unreadable" in reservation_refusal([value], [_value()]) + + +def test_known_defaults_and_unknown_fields_survive_validation() -> None: + from server.router.fingerprints import reservation_refusal + + old = {"v": 1, "fields": {}} + newer = {"v": 1, "fields": {"future": {"items": [1, True, None]}}} + before = deepcopy(newer) + assert reservation_refusal([old], [newer]) == "" + assert newer == before + + +def test_cycle_and_non_json_unknowns_fail_closed() -> None: + from server.router.fingerprints import reservation_refusal + + cyclic = _value() + cyclic["fields"]["future"] = cyclic + assert "unreadable" in reservation_refusal([_value()], [cyclic]) + assert "unreadable" in reservation_refusal([_value()], [_value(future=object())]) + + +def test_candidate_fits_a_live_placement_with_the_largest_org() -> None: + import json + + from server.router.fingerprints import placement_refusal + from server.router.limits import MAX_BODY_BYTES + + value = _value(future="") + overhead = len(json.dumps({"org": "", "cell": "first", "fingerprint": value})) + room = MAX_BODY_BYTES - overhead - 6 * cell.ORG_MAX_BYTES + value["fields"]["future"] = "x" * room + assert placement_refusal("first", "candidate", value) == "" + value["fields"]["future"] += "x" + reason = placement_refusal("first", "candidate", value) + assert "future" in reason + assert "x" * 20 not in reason From 9ac5f0ef673169d7dafea3eb376fce1d1848d0fd Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:46:28 +0000 Subject: [PATCH 049/181] Keep target settings values out of update errors Signed-off-by: zlareb1 --- server/cli/cell_update_config.py | 22 ++++++++++----- tests/unit/test_cell_update_config.py | 40 +++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/server/cli/cell_update_config.py b/server/cli/cell_update_config.py index 26ee346e7f..ce9e6452f2 100644 --- a/server/cli/cell_update_config.py +++ b/server/cli/cell_update_config.py @@ -30,16 +30,24 @@ def settings_customise_sources( def read_target(path: Path, current: Settings) -> Settings: """Overlay FILE on the CLI's effective settings; install neither the file nor values.""" - values = read_settings_file(str(path), set(Settings.model_fields)) + try: + values = read_settings_file(str(path), set(Settings.model_fields)) + except (SettingsFileError, UnicodeError, TypeError): + # Reader messages may quote a version, YAML source, or an arbitrary mapping key. + raise SettingsFileError( + f"{path}: invalid target settings file; use version: 1, unique setting names " + "and nonsecret values" + ) from None checked: dict[str, Any] = {} for name, value in values.items(): try: checked[name] = coerce(Settings, name, value) - except ValidationError as error: - rule = error.errors(include_input=False)[0]["msg"] - raise SettingsFileError(f"{path}: {name} has an invalid value ({rule})") from None + except ValidationError: + raise SettingsFileError(f"{path}: {name} has an invalid target value") from None try: return _Target(**(current.model_dump() | checked)) - except ValidationError as error: - # Settings hides inputs, including a model-level validator's complete input. - raise SettingsFileError(f"{path}: invalid target configuration ({error})") from None + except ValidationError: + # Validators can embed supplied values in their own messages despite hidden inputs. + raise SettingsFileError( + f"{path}: invalid target configuration; check the setting constraints" + ) from None diff --git a/tests/unit/test_cell_update_config.py b/tests/unit/test_cell_update_config.py index fd1721ec42..8fbcb32444 100644 --- a/tests/unit/test_cell_update_config.py +++ b/tests/unit/test_cell_update_config.py @@ -57,7 +57,10 @@ def test_invalid_overlay_fields_refuse_without_echoing_values( path.write_text("version: 1\nsettings:\n" + body) with pytest.raises(SettingsFileError) as exc: read_target(path, current) - assert field in str(exc.value) + if field == "memory_ask_deep_evidence_max_memories": + assert field in str(exc.value) + else: + assert "invalid target settings file" in str(exc.value) assert "do-not-copy-this-value" not in str(exc.value) @@ -74,7 +77,7 @@ def test_full_model_rules_apply_and_unknown_settings_keep_the_normal_warning( path.write_text( "version: 1\nsettings:\n inference_token_audience: own-audience\n public_base_url: ''\n" ) - with pytest.raises(SettingsFileError, match="LOOM_INFERENCE_TOKEN_URL"): + with pytest.raises(SettingsFileError, match="invalid target configuration"): read_target(path, current) @@ -136,3 +139,36 @@ def test_explicit_fingerprint_is_identical_to_the_process_and_uses_candidate_org assert "target-repo" in changed["configured_orgs"] assert "before" not in changed["configured_orgs"] assert cell.fingerprint("deployment") == fields + + +@pytest.mark.parametrize( + "template", + [ + "version: 1\nsettings:\n openai_base_url: {marker}\n", + "version: {marker}\nsettings: {{}}\n", + "version: 1\nsettings:\n jwt_secret: [\n {marker}\n", + "version: 1\nsettings:\n memory_ask_deep_evidence_max_memories: {marker}\n", + "version: 1\nsettings:\n {marker}: first\n {marker}: second\n", + "version: 1\nsettings:\n tracker_org: 'next'\n{marker}: extra\n", + ], +) +def test_invalid_target_errors_never_echo_supplied_values(tmp_path: Path, template: str) -> None: + from server.cli.cell_update_config import read_target + + marker = "private-overlay-regression-marker" + path = tmp_path / "target.yaml" + path.write_text(template.format(marker=marker)) + with pytest.raises(SettingsFileError) as error: + read_target(path, _config()) + assert marker not in str(error.value) + assert error.value.__suppress_context__ + assert "target" in str(error.value) + + +def test_malformed_target_encoding_is_a_safe_configuration_refusal(tmp_path: Path) -> None: + from server.cli.cell_update_config import read_target + + path = tmp_path / "target.yaml" + path.write_bytes(b"version: 1\nsettings:\n tracker_org: \xff\n") + with pytest.raises(SettingsFileError, match="invalid target settings file"): + read_target(path, _config()) From f4702f64809e33a2f4c01a312fa99b82a64131e9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:47:36 +0000 Subject: [PATCH 050/181] Use one owner for native cell control sessions Supervise attach, prepare, removal and backout on their exact lock backend. Keep cleanup inside phase deadlines, forward fatal guard exceptions after close, and retry mode contention only after acknowledged rollback and ownership checks. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 2 + docs/architecture/cells.md | 6 +- server/cli/cell_attach.py | 131 ++++------- server/cli/cell_attach_pg.py | 206 +++++++----------- server/cli/cell_control.py | 2 + tests/integration/test_cell_attach.py | 19 +- tests/integration/test_cell_attach_cleanup.py | 28 +-- .../integration/test_cell_control_cleanup.py | 79 +++++++ tests/integration/test_cell_control_fence.py | 30 ++- tests/unit/cell_control_owner.py | 11 + tests/unit/test_cell_attach.py | 65 +++--- tests/unit/test_cell_control.py | 150 +++++++++++++ tests/unit/test_cell_control_census.py | 55 +++-- tests/unit/test_cell_prepare.py | 1 + tests/unit/test_router_attach.py | 13 +- 15 files changed, 503 insertions(+), 295 deletions(-) create mode 100644 tests/unit/cell_control_owner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3066d67299..3b5b7aced8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,6 +902,8 @@ jobs: tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_attach_cleanup.py + tests/integration/test_cell_control_cleanup.py + tests/integration/test_cell_control_fence.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 70a203783a..c0a4115499 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1263,6 +1263,10 @@ cleanup. An unexpected transaction error propagates immediately; it never waits rollback on a session the owner will discard. Uncapped command work retains the existing one-second session-exit allowance, which cannot extend an expired bounded phase. A transport abort remains an unknown outcome, not proof that the server backend ended. +Mode-lock contention retries only after an acknowledged pre-COMMIT rollback and a check +that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check failure +never becomes a busy retry. After attempting the mode update, the diagnostic stays +unknown until rollback plus ownership or COMMIT is acknowledged. The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded @@ -1271,7 +1275,7 @@ entry points and the named exceptions: | Writer | Fence | | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | | `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | -| `cell_attach_pg._mode_try` | Attach's claim, recording/first/fingerprint comparison, exact lock backend | +| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | | `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | | `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | | `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ed57bb88da..a8a78279fd 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -155,7 +155,7 @@ def __init__( self.name = args.cell self.wait = apg.Wait(args.wait) self.lock_conn: Any = None - self._lock_cm: Any = None + self.owner: cell_control.Owner self.row: dict[str, Any] = {} self.mode = "" self.own_fp: tuple[dict[str, Any] | None, dict[str, Any] | None] = (None, None) @@ -165,35 +165,6 @@ def __init__( # -- the lock session -------------------------------------------------------------- - async def open_lock(self) -> None: - self._lock_cm = cell_control.session("attach") - self.lock_conn = await self._lock_cm.__aenter__() - - async def close_lock(self) -> None: - from server.core import pg - - if self._lock_cm is not None: - cm, conn = self._lock_cm, self.lock_conn - self._lock_cm = self.lock_conn = None - closed = pg._lost(conn) - end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) - try: - if not closed: - await pg.close_owned(conn, end=end) - finally: - task = asyncio.create_task(cm.__aexit__(None, None, None)) - task.add_done_callback(pg._retrieve_task) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - if not done: - task.cancel() - raise TimeoutError("attach lock cleanup deadline expired") - task.result() - except BaseException: - pg._terminate(conn) - task.cancel() - raise - async def confirm_lock(self) -> None: """A lost backend ends the owner; a new run must claim and start again.""" await cell_control.check_lock(self.lock_conn) @@ -417,7 +388,7 @@ def _named(self, found: cell_scan.ScanResult) -> dict[str, Any]: } async def triggers(self) -> None: - if refusals := await apg.ensure_triggers(self.lock_conn, self.wait, self.log): + if refusals := await apg.ensure_triggers(self.owner, self.wait, self.log): raise Refused(refusals) # -- step 3 ------------------------------------------------------------------------ @@ -437,7 +408,7 @@ def mark(mode: str) -> None: self.mode = mode await apg.commit_enforced( - self.lock_conn, + self.owner, self.confirm_lock, self.ready_for_commit, self.wait, @@ -448,32 +419,16 @@ def mark(mode: str) -> None: ) async def durable(self) -> int: - """The durable read of `enforced`, retried within `--wait`; then X, the snapshot's + """The bounded durable read of `enforced`; then X, the snapshot's xmax, above the read's own xid.""" - from server.core import pg, pg_profile + from server.core import pg_profile with self.wait.counting(): - while True: - if await pg_profile.standby_now() != "synchronous": - raise _refused("postgres standbys", "not synchronous before the durable read") - try: - read = await apg.durable_read(self.lock_conn) - except Exception as e: - # A try cut off by its bound, or an outage: its outcome is unknown, and - # the next try reads again. Anything else is no reason to retry. - outage = pg.is_outage(e) or isinstance(e, pg.PostgresUnavailableError) - if not (isinstance(e, TimeoutError) or outage): - raise - await self.confirm_lock() - self.log(f"the durable read failed ({type(e).__name__}); retrying") - if not await self.wait.pause(): - raise _refused( - "postgres durable read", "did not commit within --wait" - ) from e - continue - if await pg_profile.standby_now() != "synchronous": - raise _refused("postgres standbys", "not synchronous after the durable read") - break + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous before the durable read") + read = await apg.durable_read(self.owner) + if await pg_profile.standby_now() != "synchronous": + raise _refused("postgres standbys", "not synchronous after the durable read") if read.mode in cell.MODES: self.mode = read.mode if read.first != (True, self.name) or read.mode != "enforced": @@ -608,44 +563,36 @@ async def run(self) -> int: await self.done() self.report.update(state="serving", done=True) return 0 - failure: BaseException | None = None - try: - await self.open_lock() - await self.read_own_row() - topology, names, listed = await self.refusals() - if self.args.dry_run: - return await self.dry_run(topology, listed) - self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) - await self.begin() - state, _ = await apg.read_mode(self.lock_conn) - self.mode = state + return await cell_control.run("attach", self._owned) + + async def _owned(self, owner: cell_control.Owner) -> int: + self.owner, self.lock_conn = owner, owner.conn + await self.read_own_row() + topology, names, listed = await self.refusals() + if self.args.dry_run: + return await self.dry_run(topology, listed) + self.row = await cell_control.claim(self.lock_conn, self.router.call, self.row) + await self.begin() + state, _ = await apg.read_mode(self.lock_conn) + self.mode = state + await self.triggers() + if state == "recording": + found = await self.scan(topology, listed) + if found.too_long: + raise Refused(_too_long(found)) + written = await apg.write_tenancy(found.orgs) + self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") await self.triggers() - if state == "recording": - found = await self.scan(topology, listed) - if found.too_long: - raise Refused(_too_long(found)) - written = await apg.write_tenancy(found.orgs) - self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") - await self.triggers() - await self.precommit(found) - await self.commit_mode() - else: - await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) - xmax = await self.durable() - await self.check_beats("ignore") - await self.step4(xmax, topology, names) - answer = await self.serving() - self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) - return 0 - except BaseException as error: - failure = error - raise - finally: - try: - await self.close_lock() - except BaseException: - if failure is None: - raise + await self.precommit(found) + await self.commit_mode() + else: + await self.mutate("/attach/enforced", {"deployment_id": self.row["deployment_id"]}) + xmax = await self.durable() + await self.check_beats("ignore") + await self.step4(xmax, topology, names) + answer = await self.serving() + self.report.update(state=answer.get("state"), placed_orgs=answer.get("placed_orgs")) + return 0 async def precommit(self, found: cell_scan.ScanResult) -> None: from server.core import pg_profile diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d24f749996..d18ff6f628 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -13,14 +13,17 @@ import contextlib import json import time -from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator +from collections.abc import Awaitable, Callable, Iterable, Iterator from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any from server.cli.cell_ch import Refusal, Refused from server.core import cell, cell_triggers, pg from server.core.cell import ATTACH_LOCK_KEY +if TYPE_CHECKING: + from server.cli.cell_control import Owner + _sleep = asyncio.sleep # the seam the wait tests replace # The pause between polls of a wait. @@ -90,50 +93,6 @@ async def pause(self, seconds: float = POLL_S) -> bool: return True -@dataclass -class _Interrupted: - error: BaseException - - -async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: - try: - return await work - except (KeyboardInterrupt, SystemExit) as error: - # These exceptions otherwise stop asyncio's loop before the owner can close. - return _Interrupted(error) - - -async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) -> T: - """Bound owned work, reserving cleanup inside its original phase deadline. - - A failed borrowed transaction must remain alive until public close sends its - cancellation. Closing the transport first can leave its backend locks held. - """ - task = asyncio.create_task(_work_result(work)) - reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) - if not done: - raise TimeoutError("attach Postgres phase deadline expired") - result = task.result() - if isinstance(result, _Interrupted): - raise result.error - return result - except BaseException: - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the work failure or cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. - pg._terminate(conn) - raise - - def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( @@ -500,12 +459,13 @@ async def prepared_refusals(conn: Any) -> list[Refusal]: _MIGRATION_UNLOCK = "SELECT pg_advisory_unlock($1)" -async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: +async def ensure_triggers(owner: Owner, wait: Wait, log: Callable[[str], None]) -> list[Refusal]: """Give every planned table its triggers (`cell_triggers.ensure_table` with `recording_first`), one try at a time under the migration lock, retrying a table whose lock is busy a second apart within `wait`; one autovacuum holding it is asked to yield by one try just past `deadlock_timeout`. A table this role cannot alter refuses at once. Then `missing`: any defect refuses.""" + conn = owner.conn deadlock_ms = int( await conn.fetchval( "SELECT setting::int * CASE unit WHEN 's' THEN 1000 ELSE 1 END " @@ -514,14 +474,15 @@ async def ensure_triggers(conn: Any, wait: Wait, log: Callable[[str], None]) -> ) with wait.counting(): for table in await cell_triggers.planned_tables(conn): - if refusal := await _ensure_table(conn, table, deadlock_ms, wait, log): + if refusal := await _ensure_table(owner, table, deadlock_ms, wait, log): return [refusal] return [Refusal("postgres triggers", d.text) for d in await cell_triggers.missing(conn)] async def _ensure_table( - conn: Any, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] + owner: Owner, table: str, deadlock_ms: int, wait: Wait, log: Callable[[str], None] ) -> Refusal | None: + conn = owner.conn yielded = False said = "" while True: @@ -529,7 +490,7 @@ async def _ensure_table( holders = await cell_triggers.holders(conn, table) if not yielded and any(h["backend_type"] == "autovacuum worker" for h in holders): most_ms, yielded = deadlock_ms + 500, True - outcome = await _ensure_one(conn, table, most_ms, wait) + outcome = await _ensure_one(owner, table, most_ms, wait) if outcome.answer == "applied": return None if outcome.answer == "denied": @@ -567,7 +528,7 @@ async def _migration_holder(conn: Any, key: int) -> str: return f"{_MIGRATION_HELD} (held by {held or 'a session that just ended'})" -async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: +async def _ensure_one(owner: Owner, table: str, most_ms: int, wait: Wait) -> cell_triggers.Outcome: """One try under the migration lock: the migration lock's wait, and then the DDL's lock wait (at most `most_ms`), each bounded by what `wait` has left when it starts, with the try floor. One client deadline covers the lock, DDL and cleanup, including COMMIT or @@ -577,8 +538,8 @@ async def _ensure_one(conn: Any, table: str, most_ms: int, wait: Wait) -> cell_t bound_s = ( wait.bound_ms(_MIGRATION_LOCK_MS) + wait.bound_ms(most_ms) ) / 1000 + _TRIGGER_TRY_MARGIN_S - return await supervise( - conn, _trigger_try(conn, table, most_ms, wait, _LOCK_KEY), end=time.monotonic() + bound_s + return await owner.call( + lambda: _trigger_try(owner.conn, table, most_ms, wait, _LOCK_KEY), total_s=bound_s ) @@ -627,7 +588,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: async def commit_enforced( - conn: Any, + owner: Owner, confirm_lock: Callable[[], Any], ready: Callable[[], Awaitable[list[str]]], wait: Wait, @@ -643,16 +604,11 @@ async def commit_enforced( await confirm_lock() booting = await ready() if not booting: - try: - done = await _mode_try( - conn, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp - ) - except Refused: - mark("recording") # the UPDATE's transaction rolled back - raise + done = await _mode_try( + owner, wait.bound_ms(_MODE_LOCK_MS), mark, expected_cell, expected_fp + ) if done: return - mark("recording") log("the mode's lock is busy; retrying") if not await wait.pause(): if booting: @@ -674,65 +630,69 @@ async def commit_enforced( async def _mode_try( - conn: Any, + owner: Owner, lock_ms: int, mark: Callable[[str], None], expected_cell: str, expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], ) -> bool: - """One try including rollback or COMMIT, bounded by its lock and statement waits plus - the standby allowance; False after a statement timeout rolls back, TimeoutError at - the try's deadline.""" + """Keep work, acknowledged contention rollback and cleanup in one try budget.""" bound_s = (lock_ms + _MODE_STATEMENT_MARGIN_MS) / 1000 + _MODE_STANDBY_S + return await owner.call( + lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), + total_s=bound_s, + ) + + +async def _mode_transaction( + conn: Any, + lock_ms: int, + mark: Callable[[str], None], + expected_cell: str, + expected_fp: tuple[dict[str, Any] | None, dict[str, Any] | None], +) -> bool: + from server.cli import cell_control + + tx = conn.transaction() + await tx.start() + updated = False try: - async with asyncio.timeout(bound_s): - tx = conn.transaction() - await tx.start() - try: - await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") - await conn.execute( - f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" - ) - from server.cli import cell_control - - await cell_control.check_lock(conn) - await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") - state, first = await read_mode(conn) - own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - observed = None if own is None else (own["current_fp"], own["next_fp"]) - if ( - state != "recording" - or first != (True, expected_cell) - or json.dumps(observed, sort_keys=True) - != json.dumps(expected_fp, sort_keys=True) - ): - raise Refused( - [Refusal("postgres acceptance rows", "changed: stop and rerun attach")] - ) - mark("unknown") - status = await conn.execute( - "UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()" - ) - if status != "UPDATE 1": - raise Refused( - [Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")] - ) - except Exception as e: - await tx.rollback() - if getattr(e, "sqlstate", None) in _TIMED_OUT: - return False - raise - await tx.commit() - return True - except BaseException: - # Termination is synchronous: cancellation never starts an unbounded rollback. - conn.terminate() - raise + await conn.execute(f"SET LOCAL lock_timeout = '{lock_ms}ms'") + await conn.execute( + f"SET LOCAL statement_timeout = '{lock_ms + _MODE_STATEMENT_MARGIN_MS}ms'" + ) + await cell_control.check_lock(conn) + await conn.execute("LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE") + state, first = await read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + observed = None if own is None else (own["current_fp"], own["next_fp"]) + if ( + state != "recording" + or first != (True, expected_cell) + or json.dumps(observed, sort_keys=True) != json.dumps(expected_fp, sort_keys=True) + ): + raise Refused([Refusal("postgres acceptance rows", "changed: stop and rerun attach")]) + mark("unknown") + updated = True + status = await conn.execute("UPDATE loom.tenancy_mode SET mode = 'enforced', since = now()") + if status != "UPDATE 1": + raise Refused([Refusal("postgres loom.tenancy_mode", f"the mode update read {status}")]) + except Exception as error: + if getattr(error, "sqlstate", None) not in _TIMED_OUT: + raise + await tx.rollback() + await cell_control.check_lock(conn) + if updated: + mark("recording") + return False + await tx.commit() + mark("enforced") + return True # -- the durable read ------------------------------------------------------------------- -_DURABLE_TRY_S = 10.0 # not a setting: one try's bound; a cancelled one is retried +_DURABLE_TRY_S = 10.0 # not a setting: the state/WAL/COMMIT phase includes its cleanup @dataclass(frozen=True) @@ -745,24 +705,26 @@ class Durable: xid: int -async def durable_read(conn: Any) -> Durable: - """The mode and the `first` row read in a transaction that emits a transactional - logical message, whose commit waits for the synchronous standbys: once it returns, the - mode read is on them. Refused when the message took no xid.""" +async def durable_read(owner: Owner) -> Durable: + """Read and synchronously fence state on the owner's exact ATTACH backend.""" from server.cli import cell_control - async with asyncio.timeout(_DURABLE_TRY_S), cell_control.transaction(conn): + conn = owner.conn + + async def read() -> Durable: await conn.execute("SELECT pg_logical_emit_message(true, 'loom.attach', '')") xid = await conn.fetchval("SELECT pg_current_xact_id_if_assigned()::text") mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") - if xid is None: - raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) - return Durable( - mode=str(mode or ""), - first=None if first is None else (bool(first["is_first"]), str(first["cell"])), - xid=int(xid), - ) + if xid is None: + raise Refused([Refusal("postgres durable read", "its logical message took no xid")]) + return Durable( + mode=str(mode or ""), + first=None if first is None else (bool(first["is_first"]), str(first["cell"])), + xid=int(xid), + ) + + return await owner.call(lambda: cell_control.transaction(conn, read), total_s=_DURABLE_TRY_S) async def snapshot_xmax(conn: Any) -> int: diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 84661747fd..7fd92e28ab 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -239,6 +239,8 @@ async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: raise try: await owner.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None finally: await owner._stop(set(owner._tasks)) return result diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 573d8fc85a..6cbc595bd5 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -35,6 +35,7 @@ from server.router import directory from server.settings import settings from tests.integration._pg_database import create_database, drop_database +from tests.unit.cell_control_owner import owner as _owner OPERATOR = "lco_" + "2" * 128 CREDENTIAL_SHA = hashlib.sha256(b"lcr_first").hexdigest() @@ -535,7 +536,7 @@ async def release() -> None: await cell_attach_pg.take_lock(conn, "mode test") own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") await cell_attach_pg.commit_enforced( - conn, + _owner(conn), ok, ok, cell_attach_pg.Wait(20), @@ -812,19 +813,17 @@ async def test_signaling_another_role_requires_an_effective_signal_grant(cell_st async def test_attach_needs_its_predefined_capabilities_without_set_role( cell_stores, monkeypatch, capsys, capability: str ) -> None: - from contextlib import asynccontextmanager - stores, network = cell_stores, _Network() await _prepare(stores, network) role = f"it_caps_{uuid.uuid4().hex[:6]}" await _admin(f"CREATE ROLE {role} LOGIN") real_dedicated = pg.dedicated + real_connect = pg._connect_dedicated - @asynccontextmanager - async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: - async with real_dedicated(**kwargs) as conn: - await conn.execute(f"SET SESSION AUTHORIZATION {role}") - yield conn + async def as_attacher(**kwargs: Any) -> Any: + conn = await real_connect(**kwargs) + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + return conn try: async with real_dedicated() as conn: @@ -836,7 +835,7 @@ async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) before = await _row() with monkeypatch.context() as own: - own.setattr(pg, "dedicated", as_attacher) + own.setattr(pg, "_connect_dedicated", as_attacher) assert await _attach(stores, network) == 1 assert ( f"attach's role lacks effective privileges of {capability}" in capsys.readouterr().err @@ -848,7 +847,7 @@ async def as_attacher(**kwargs: Any) -> AsyncIterator[Any]: async with real_dedicated() as conn: await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT TRUE") with monkeypatch.context() as own: - own.setattr(pg, "dedicated", as_attacher) + own.setattr(pg, "_connect_dedicated", as_attacher) assert await _attach(stores, network) == 0, capsys.readouterr().err assert (await _row()).state == "serving" finally: diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 145be1188d..3b65c81b7d 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -1,17 +1,16 @@ """Attach owners close active sessions before cancelling transaction work.""" -import argparse import asyncio -import contextlib import time import pytest -from server.cli import cell_attach from server.cli import cell_attach_pg as apg +from server.cli import cell_control from server.core import cell_triggers, pg from tests.integration.test_pg_owned_cleanup import database as database from tests.integration.test_pg_owned_cleanup import released, sleeping +from tests.unit.cell_control_owner import owner as _owner pytestmark = pytest.mark.asyncio @@ -52,7 +51,7 @@ async def intercepted(self, query, *args, **kwargs): if owner == "mode" and query.startswith("LOCK TABLE"): await execute(self, "SELECT * FROM probe FOR UPDATE") if phase == "rollback": - raise ValueError("mode statement failed") + raise asyncpg.LockNotAvailableError("mode statement busy") return await execute(self, "SELECT pg_sleep(10)") if owner == "direct" and query.startswith("BEGIN"): await execute(self, query) @@ -73,9 +72,8 @@ async def ensure(conn, table, **kwargs): conn, table, statements, kwargs["lock_timeout"], 1, warn=False ) - @contextlib.asynccontextmanager - async def dedicated(**_): - yield conn + async def connect(**_): + return conn monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) monkeypatch.setattr(asyncpg.Connection, "close", closing_session) @@ -83,15 +81,19 @@ async def dedicated(**_): monkeypatch.setattr(apg, "_TRIGGER_TRY_MARGIN_S", 0.4) monkeypatch.setattr(apg, "_MODE_STANDBY_S", 0.5) monkeypatch.setattr(apg, "_MODE_STATEMENT_MARGIN_MS", 0) - monkeypatch.setattr(cell_attach, "_LOCK_STATEMENT_MS", 600) - monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_control, "OPEN_S", 0.6) + monkeypatch.setattr(pg, "_connect_dedicated", connect) + await apg.take_lock(conn, "cleanup test") if owner == "trigger": - work = apg._ensure_one(conn, "probe", 100, apg.Wait(0)) + work = apg._ensure_one(_owner(conn), "probe", 100, apg.Wait(0)) elif owner == "mode": - work = apg._mode_try(conn, 100, lambda _: None) + work = apg._mode_try(_owner(conn), 100, lambda _: None, "first", (None, None)) else: - run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) - work = run.open_lock() + + async def body(_owner): + raise AssertionError("direct checks unexpectedly completed") + + work = cell_control.run("test", body) task = asyncio.create_task(work) try: await sleeping(observer, pid) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index 2a37ca6170..f66eeb5169 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -214,3 +214,82 @@ async def main(): if child.returncode is None: child.kill() await child.wait() + + +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +async def test_mode_contention_retries_only_after_real_rollback_on_same_owned_backend( + native_database, monkeypatch, sqlstate +): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + + conn = await pg._connect_dedicated(statement_timeout_ms=10_000) + await pg.init_connection(conn) + await apg.take_lock(conn, "real mode contention") + await conn.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await conn.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('recording')") + fp = {"v": 1} + await conn.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1)", fp) + pid, marks, injected = conn.get_server_pid(), [], False + execute = asyncpg.Connection.execute + + async def contend(self, query, *args, **kwargs): + nonlocal injected + if self is conn and query.startswith("UPDATE loom.tenancy_mode") and not injected: + injected = True + await execute( + self, + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE='" + + sqlstate + + "', MESSAGE='test contention'; END $$", + ) + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", contend) + owned = owner(conn) + try: + assert not await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + assert await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" + assert marks == ["unknown", "recording"] and not owned.closed + assert await apg._mode_try(owned, 100, marks.append, "first", (fp, None)) + assert await native_database.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced" + assert marks == ["unknown", "recording", "unknown", "enforced"] + assert conn.get_server_pid() == pid and await apg.lock_held(conn) + finally: + await owned.close() + await released(native_database, pid) + + +async def test_native_attach_run_body_uses_sole_owner_on_cancellation(native_database, monkeypatch): + from uuid import UUID + + from server.cli import cell_attach + from server.router.directory import CONTROL_FIELDS + + observed, backend = asyncio.Event(), {} + + class Router: + async def call(self, _path): + return { + **dict.fromkeys(CONTROL_FIELDS, ""), + "state": "attaching", + "mutation_generation": str(UUID(int=1)), + } + + async def read(self): + backend["pid"] = self.lock_conn.get_server_pid() + observed.set() + await self.lock_conn.execute("BEGIN; SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10)") + + monkeypatch.setattr(cell_attach.Run, "read_own_row", read) + run = cell_attach.Run(None, argparse.Namespace(cell="first", wait=0), Router(), lambda _: None) + task = asyncio.create_task(run.run()) + await observed.wait() + await sleeping(native_database, backend["pid"]) + task.cancel("native attach cancellation") + with pytest.raises(asyncio.CancelledError, match="native attach cancellation"): + await asyncio.wait_for(task, 1.3) + await released(native_database, backend["pid"]) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index d96f170fa3..fa375eb12a 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -17,6 +17,7 @@ from server.core import cell from server.router import app, directory from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner TOKEN = "lco_fence-test" @@ -222,7 +223,7 @@ async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, wr with pytest.raises( (asyncpg.InterfaceError, asyncpg.PostgresError, asyncpg.InternalClientError) ): - await apg._mode_try(dead, 100, lambda _m: None, "first", (fp, None)) + await apg._mode_try(_owner(dead), 100, lambda _m: None, "first", (fp, None)) assert await admin.fetchval("SELECT mode FROM loom.tenancy_mode") == "recording" finally: await successor.close() @@ -232,7 +233,7 @@ async def test_dead_local_lock_backend_cannot_write_acceptance(local_control, wr @pytest.mark.asyncio async def test_same_session_prepare_and_mode_commit(local_control) -> None: from server.cli import cell_attach_pg as apg - from server.cli import cell_control, cell_prepare + from server.cli import cell_prepare conn, _ = local_control await apg.take_lock(conn, "test") @@ -243,8 +244,8 @@ async def test_same_session_prepare_and_mode_commit(local_control) -> None: ) assert written == ["first", "mode", "own"] assert await apg.lock_held(conn) - assert await apg._mode_try(conn, 100, lambda _m: None, "first", (fp, None)) - result = await apg.durable_read(conn) + assert await apg._mode_try(_owner(conn), 100, lambda _m: None, "first", (fp, None)) + result = await apg.durable_read(_owner(conn)) assert result.mode == "enforced" and result.first == (True, "first") assert await apg.lock_held(conn) # No-op prepare remains a same-backend synchronous durability boundary. @@ -253,8 +254,10 @@ async def test_same_session_prepare_and_mode_commit(local_control) -> None: await cell_prepare._write_rows(conn, rows, {"is_first": True, "mode": "enforced"}, fp, None) == [] ) - async with cell_control.transaction(conn): - assert await conn.fetchval("SELECT pg_backend_pid()") == conn.get_server_pid() + assert ( + await _owner(conn).transaction(lambda: conn.fetchval("SELECT pg_backend_pid()")) + == conn.get_server_pid() + ) @pytest.mark.asyncio @@ -334,7 +337,6 @@ async def test_inflight_finalizer_cannot_release_after_new_owner_rewidens( from starlette.routing import Route from server.cli import cell_attach_pg as apg - from server.cli import cell_control client, directory_admin, _ = control_stores local_admin, local_dsn = local_control @@ -387,9 +389,11 @@ async def change(): await local_admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) await apg.take_lock(b, "new controller") newer = await claim(client, await read(client)) + # New owner claims BEFORE widening local acceptance, on the owned backend. - async with cell_control.transaction(b): + async def widen(): await b.execute("UPDATE loom.cell_self SET current_fp = $1, next_fp = $2", current, nxt) + if aba: await directory_admin.execute( "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL" @@ -399,6 +403,7 @@ async def change(): json.dumps(current), json.dumps(nxt), ) + await _owner(b).transaction(widen) assert newer["mutation_generation"] != old["mutation_generation"] resume.set() response = await task @@ -473,7 +478,7 @@ async def wait_for_lock(key): await asyncio.sleep(0.01) else: pytest.fail("placement did not acquire its org lock") - cleanup = asyncio.create_task(cell_control.backout(local_admin, call, original)) + cleanup = asyncio.create_task(cell_control.backout(_owner(local_admin), call, original)) for _ in range(100): waiting = await directory_admin.fetchval( "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() " @@ -498,7 +503,7 @@ async def wait_for_lock(key): directory.place("queued", "first", app.placement_refusal) ) await wait_for_lock(directory._PLACE_LOCK) - await cell_control.backout(local_admin, call, original) + await cell_control.backout(_owner(local_admin), call, original) assert not await local_admin.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") await tx.commit() with pytest.raises(directory.PlacementRefused, match="closed"): @@ -580,6 +585,7 @@ async def call(self, path, body): None, argparse.Namespace(cell="first", wait=0), Router(), lambda _text: None ) run.lock_conn = a + run.owner = _owner(a) run.mode = "recording" run.own_fp = (fp, None) run.row = { @@ -658,7 +664,7 @@ async def call(path, body): placement_error = str(exc) return answer - await cell_control.backout(local, call, await read(client)) + await cell_control.backout(_owner(local), call, await read(client)) assert opened.status_code == 409, opened.text assert "serving" in opened.json()["error"] assert placement_error and "closed" in placement_error @@ -788,7 +794,7 @@ async def call(path, body): await asyncio.wait_for(entered.wait(), 3) await local.execute("SELECT pg_terminate_backend($1)", old_backend.get_server_pid()) await apg.take_lock(local, "backout successor") - await cell_control.backout(local, call, await read(client)) + await cell_control.backout(_owner(local), call, await read(client)) assert await apg.lock_held(local) assert not await local.fetchval("SELECT EXISTS(SELECT 1 FROM loom.cell_self)") assert not await directory_admin.fetchval( diff --git a/tests/unit/cell_control_owner.py b/tests/unit/cell_control_owner.py new file mode 100644 index 0000000000..3a2d009821 --- /dev/null +++ b/tests/unit/cell_control_owner.py @@ -0,0 +1,11 @@ +"""Provide an explicit owner for a test's already-open raw session.""" + +from typing import Any + +from server.cli.cell_control import Owner + + +def owner(conn: Any) -> Owner: + owned = Owner() + owned.conn = conn + return owned diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 0b3401be88..a69c5c5c8f 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -8,7 +8,6 @@ import argparse import asyncio import json -import time from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -16,7 +15,7 @@ import httpx import pytest -from server.cli import cell_attach, cell_scan +from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg from server.cli.cell_ch import Refusal, Refused @@ -24,6 +23,7 @@ from server.core import cell, instances from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings +from tests.unit.cell_control_owner import owner as _owner FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} CURRENT = cell.as_value(FIELDS) @@ -253,7 +253,7 @@ async def ready() -> list[str]: clock.now += 100 # a scan before the wait with pytest.raises(Refused, match="not taken within --wait"): await apg.commit_enforced( - conn, + _owner(conn), nothing, ready, wait, @@ -287,7 +287,7 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = FakeConn([("deadlock_timeout", 1000)]) wait = apg.Wait(60, clock) - (refusal,) = await apg.ensure_triggers(conn, wait, lambda _t: None) + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, lambda _t: None) assert refusal.subject == "postgres loom.scope_generations" assert "cannot alter it" in refusal.reason and wait.spent < 1 @@ -300,7 +300,7 @@ async def busy(*_a: Any, **_kw: Any) -> Any: monkeypatch.setattr(cell_triggers, "ensure_table", busy) wait = apg.Wait(10, clock) logged: list[str] = [] - (refusal,) = await apg.ensure_triggers(conn, wait, logged.append) + (refusal,) = await apg.ensure_triggers(_owner(conn), wait, logged.append) # The last try ran with the floor's lock wait, after --wait was all but spent. assert "not taken within --wait" in refusal.reason assert wait.spent <= 10 + apg.TRY_FLOOR_MS / 1000 + 1e-9 @@ -519,9 +519,6 @@ async def __aexit__(self, *exc: Any) -> None: return _Tx() - async def close(self, *, timeout: float) -> None: - self.executed.append("CLOSE") - def terminate(self) -> None: self.executed.append("TERMINATE") @@ -534,14 +531,14 @@ def is_closed(self) -> bool: def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: - supervise = apg.supervise + supervise = cell_control.Owner.call - async def bounded(conn: Any, work: Any, *, end: float) -> Any: + async def bounded(self: Any, work: Any, *, total_s: float) -> Any: if bounds is not None: - bounds.append(end - time.monotonic()) - return await supervise(conn, work, end=time.monotonic() + 0.01) + bounds.append(total_s) + return await supervise(self, work, total_s=0.01) - monkeypatch.setattr(apg, "supervise", bounded) + monkeypatch.setattr(cell_control.Owner, "call", bounded) def _seq(*values: Any) -> Callable[..., Any]: @@ -722,7 +719,7 @@ async def ready() -> list[str]: logged: list[str] = [] await apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(10, clock), @@ -747,6 +744,7 @@ async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( ) -> None: conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.own_fp = (CURRENT, NEXT) async def read(_ch: Any) -> list[beats.Beat]: @@ -759,7 +757,7 @@ async def confirm() -> None: with pytest.raises(Refused) as error: await apg.commit_enforced( - conn, + _owner(conn), confirm, run.ready_for_commit, apg.Wait(wait_s, clock), @@ -801,7 +799,7 @@ async def ready() -> list[str]: with pytest.raises(TimeoutError): await asyncio.wait_for( apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(wait_s, clock), @@ -837,7 +835,9 @@ async def execute(self, sql: str, *args: Any) -> str: _short_phases(monkeypatch) marks: list[str] = [] task = asyncio.create_task( - apg._mode_try(conn, 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT)) + apg._mode_try( + _owner(conn), 100, marks.append, expected_cell="first", expected_fp=(CURRENT, NEXT) + ) ) done, _ = await asyncio.wait([task], timeout=0.2) if not done: @@ -905,9 +905,9 @@ async def confirm() -> None: async def ready() -> list[str]: return [] - with pytest.raises(KeyboardInterrupt): + with pytest.raises(cell_control._ForwardFatal) as fatal: await apg.commit_enforced( - conn, + _owner(conn), confirm, ready, apg.Wait(2, clock), @@ -916,6 +916,7 @@ async def ready() -> list[str]: expected_cell="first", expected_fp=(CURRENT, NEXT), ) + assert isinstance(fatal.value.fatal.error, KeyboardInterrupt) assert conn.executed[-1] == "CLOSE" assert "TERMINATE" not in conn.executed assert marks == [] @@ -932,6 +933,7 @@ async def synchronous() -> str: monkeypatch.setattr(pg_profile, "standby_now", synchronous) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.lock_conn = FakeConn([("pg_snapshot_xmax", "100")]) async def read(mode: str, xid: int) -> apg.Durable: @@ -974,6 +976,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: monkeypatch.setattr(pg, "dedicated", dedicated) monkeypatch.setattr(apg, "read_mode", read_mode) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) assert await run.done() state[0] = "recording" with pytest.raises(Refused, match="attach lowers and repairs neither"): @@ -989,6 +992,7 @@ async def test_a_root_secret_change_during_attach_refuses_at_the_end( monkeypatch: pytest.MonkeyPatch, ) -> None: run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] + run.owner = _owner(FakeConn()) run.begun_keys_fp = "before" async def settle(*_a: Any, **_kw: Any) -> beats.Verdict: @@ -1021,6 +1025,7 @@ async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row = {"deployment_id": "dep-1"} await self.commit_mode() raise failure # what stops the run before the durable read finishes @@ -1086,7 +1091,7 @@ async def ensure_table(*_a: Any, lock_timeout: str, **_kw: Any) -> Any: conn = FakeConn([("pg_advisory_lock", migration_lock), ("pg_advisory_unlock", "OK")]) wait = apg.Wait(2, clock) with wait.counting(): - outcome = await apg._ensure_one(conn, "scope_generations", 500, wait) + outcome = await apg._ensure_one(_owner(conn), "scope_generations", 500, wait) assert outcome.answer == "applied" assert asked == [f"{apg.TRY_FLOOR_MS}ms"] @@ -1125,7 +1130,9 @@ async def ensure_table(conn: Any, table: str, **kwargs: Any) -> Any: _short_phases(monkeypatch) monkeypatch.setattr(cell_triggers, "ensure_table", ensure_table) conn = _Stalled() - task = asyncio.create_task(apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock))) + task = asyncio.create_task( + apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) + ) done, _ = await asyncio.wait([task], timeout=0.2) try: assert task in done, f"trigger deadline is stuck at {stall}" @@ -1162,7 +1169,7 @@ async def ensure_table(*_a: Any, **_kw: Any) -> Any: ] ) with pytest.raises(ValueError, match=f"{failed if failed == 'migration' else 'DDL'} failed"): - await apg._ensure_one(conn, "scope_generations", 100, apg.Wait(0, clock)) + await apg._ensure_one(_owner(conn), "scope_generations", 100, apg.Wait(0, clock)) assert conn.executed[-1] == "CLOSE" assert "TERMINATE" not in conn.executed @@ -1194,7 +1201,7 @@ async def no_holders(_c: Any, _t: str) -> list[Any]: monkeypatch.setattr(cell_triggers, "planned_tables", planned) monkeypatch.setattr(cell_triggers, "holders", no_holders) logged: list[str] = [] - (refusal,) = await apg.ensure_triggers(conn, apg.Wait(2, clock), logged.append) + (refusal,) = await apg.ensure_triggers(_owner(conn), apg.Wait(2, clock), logged.append) named = "the migration lock (held by pid 42, loom, since t0)" assert logged == [f"loom.scope_generations: its lock is busy ({named}); retrying"] assert refusal.reason == f"its triggers were not created within --wait: {named}" @@ -1224,7 +1231,7 @@ async def nothing() -> None: pass await apg.commit_enforced( - conn, + _owner(conn), nothing, ok, apg.Wait(0, clock), @@ -1233,7 +1240,7 @@ async def nothing() -> None: expected_cell="first", expected_fp=(CURRENT, NEXT), ) - assert marks == ["unknown"] + assert marks == ["unknown", "enforced"] assert any(f"lock_timeout = '{apg.TRY_FLOOR_MS}ms'" in sql for sql in conn.executed) async def planned(_c: Any) -> list[str]: @@ -1253,7 +1260,7 @@ async def no_defects(_c: Any) -> list[Any]: monkeypatch.setattr(cell_triggers, "ensure_table", applied) monkeypatch.setattr(cell_triggers, "missing", no_defects) ddl = FakeConn([("deadlock_timeout", 1000)]) - assert await apg.ensure_triggers(ddl, apg.Wait(0, clock), lambda _t: None) == [] + assert await apg.ensure_triggers(_owner(ddl), apg.Wait(0, clock), lambda _t: None) == [] async def read() -> list[beats.Beat]: return [_beat("a")] @@ -1289,8 +1296,10 @@ async def confirm(_self: Any) -> None: pass async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row, self.mode = {"deployment_id": "dep-1"}, "recording" self.lock_conn = conn + self.owner = _owner(conn) self.wait = apg.Wait(2, clock) await self.commit_mode() return 0 @@ -1334,6 +1343,7 @@ async def refuse(_self: Any, path: str, _body: Any = None) -> dict[str, Any]: raise Refused([Refusal("router", f"refused {path}")]) async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row, self.mode = {"deployment_id": "dep-1"}, "recording" await self.commit_mode() return 0 @@ -1358,6 +1368,7 @@ async def read(_conn: Any) -> apg.Durable: return apg.Durable(mode="recording", first=(True, "first"), xid=1) async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.mode = "unknown" # a mode commit whose outcome was lost await self.durable() return 0 @@ -1385,6 +1396,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: return "enforced", (True, "first") async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.row = {"is_first": True, "state": "attaching", "deployment_id": "dep-1"} self.lock_conn = FakeConn([("FROM loom.authority", "dep-1")]) if refusals := await self.stores_refusals(): @@ -1408,6 +1420,7 @@ async def test_an_interrupt_prints_the_mode_line_and_goes_on( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: async def run(self: Any) -> int: + self.owner = _owner(FakeConn()) self.mode = "unknown" raise KeyboardInterrupt diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 74ac1c130d..6b2b821b91 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -134,3 +134,153 @@ async def stalled(_request): "/claim", {}, ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sqlstate", ["55P03", "57014"]) +@pytest.mark.parametrize("where", ["UPDATE loom.tenancy_mode", "COMMIT"]) +async def test_mode_contention_is_busy_only_before_commit_after_acknowledged_rollback( + sqlstate, where +): + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT + + class Busy(Exception): + pass + + busy = Busy("expected contention") + busy.sqlstate = sqlstate + routes = [(where, busy)] + if where == "COMMIT": + routes.append(("UPDATE loom.tenancy_mode", "UPDATE 1")) + conn = FakeConn(routes) + owned = owner(conn) + marks = [] + if where == "COMMIT": + with pytest.raises(Busy) as found: + await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert found.value is busy + assert "ROLLBACK" not in conn.executed and marks == ["unknown"] + assert owned.closed + else: + assert not await apg._mode_try(owned, 100, marks.append, "first", (CURRENT, NEXT)) + assert conn.executed[-1] == "ROLLBACK" and marks == ["unknown", "recording"] + assert not owned.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed", ["rollback", "ownership", "update-status"]) +async def test_terminal_mode_failure_never_resets_unknown_or_retries(failed): + import asyncpg + + from server.cli import cell_attach_pg as apg + from tests.unit.cell_control_owner import owner + from tests.unit.test_cell_attach import CURRENT, NEXT, _seq + + original = ValueError("rollback failed") + routes = [("UPDATE loom.tenancy_mode", "UPDATE 0")] + if failed != "update-status": + routes = [("UPDATE loom.tenancy_mode", asyncpg.LockNotAvailableError("busy"))] + if failed == "rollback": + routes.append(("ROLLBACK", original)) + if failed == "ownership": + routes.append((apg._HELD, _seq(True, False))) + conn, marks = FakeConn(routes), [] + with pytest.raises(ValueError if failed == "rollback" else cell_control.Refused) as found: + await apg.commit_enforced( + owner(conn), + AsyncMock(), + AsyncMock(return_value=[]), + apg.Wait(0), + lambda _: None, + marks.append, + expected_cell="first", + expected_fp=(CURRENT, NEXT), + ) + if failed == "rollback": + assert found.value is original + assert marks == ["unknown"] and conn.executed[-1] == "CLOSE" + assert conn.executed.count("BEGIN") == 1 + + +@pytest.mark.asyncio +async def test_expired_phase_does_not_receive_another_exit_allowance(monkeypatch): + import time + + class StalledClose(FakeConn): + async def close(self, **kwargs): + self.timeouts = [kwargs["timeout"]] + self.executed.append("CLOSE START") + await asyncio.Future() + + conn = StalledClose() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.04) + + async def body(owned): + return await owned.transaction(lambda: asyncio.Future()) + + started = time.monotonic() + with pytest.raises(TimeoutError): + await cell_control.run("test", body) + assert time.monotonic() - started < 0.3 + assert conn.executed.count("CLOSE START") == 1 + assert 0 < conn.timeouts[0] <= 0.02 + assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_is_discarded_without_starting_command_work(monkeypatch): + from server.core import pg + + connected, discarded = asyncio.Event(), asyncio.Event() + + class Late(FakeConn): + def terminate(self): + super().terminate() + discarded.set() + + conn = Late() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + while not connected.is_set(): + try: + await connected.wait() + except asyncio.CancelledError: + pass + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.04) + body = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", body) + connected.set() + await asyncio.wait_for(discarded.wait(), 0.3) + body.assert_not_awaited() + assert conn.executed == ["TERMINATE"] + + +@pytest.mark.asyncio +async def test_successful_body_cannot_hide_failed_required_close(monkeypatch): + class Broken(FakeConn): + async def close(self, **_): + raise OSError("required close failed") + + conn = Broken() + + async def opening(self, _what): + self.conn = conn + + monkeypatch.setattr(cell_control.Owner, "_open", opening) + with pytest.raises(OSError, match="required close failed"): + await cell_control.run("test", AsyncMock(return_value=7)) + assert "TERMINATE" in conn.executed diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index aec52c908c..0c32387393 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -102,16 +102,16 @@ def test_cell_control_migrations_are_classified(): # Entries name every sink, including nested callbacks and dynamic table expressions. # A new sink, including another statement in an existing owner, requires classification. GUARDED_WRITERS = { - "server/cli/cell_attach_pg.py": {("_mode_try", "UPDATE", "loom.tenancy_mode"): 1}, + "server/cli/cell_attach_pg.py": {("_mode_transaction", "UPDATE", "loom.tenancy_mode"): 1}, "server/cli/cell_control.py": { - ("backout", "ALTER TABLE", "loom.cell_first"): 3, - ("backout", "DELETE FROM", "loom.{table}"): 1, - ("backout", "DROP TRIGGER", "loom.{table}"): 1, + ("backout.clear_state", "ALTER TABLE", "loom.cell_first"): 3, + ("backout.clear_state", "DELETE FROM", "loom.{table}"): 1, + ("backout.clear_triggers", "DROP TRIGGER", "loom.{table}"): 1, }, "server/cli/cell_prepare.py": { - ("_write_rows", "INSERT INTO", "loom.cell_first"): 1, - ("_write_rows", "INSERT INTO", "loom.tenancy_mode"): 1, - ("_write_rows", "INSERT INTO", "loom.cell_self"): 1, + ("_write_rows.write", "INSERT INTO", "loom.cell_first"): 1, + ("_write_rows.write", "INSERT INTO", "loom.tenancy_mode"): 1, + ("_write_rows.write", "INSERT INTO", "loom.cell_self"): 1, }, "server/router/directory.py": { ("set_limit.change", "UPDATE", "loom_router.cells"): 1, @@ -307,35 +307,54 @@ def function(path: str, name: str) -> ast.AST: @pytest.mark.parametrize( ("path", "name", "required"), [ - ("server/cli/cell_prepare.py", "attach_lock", ['cell_control.session("prepare")']), + ( + "server/cli/cell_prepare.py", + "prepare", + [ + 'cell_control.run("prepare", lambda owner: _prepare_locked(ch, args, router, transport, owner))' + ], + ), ( "server/cli/cell_prepare.py", "_prepare_locked", [ "cell_control.claim(conn, call, snapshot)", - "_write_rows(conn, rows, answer, current, nxt)", + "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=cell_control.TRANSACTION_S)", ], ), ( "server/cli/cell_prepare.py", "_write_rows", - ["cell_control.transaction(conn)", "_read_rows(conn, count_tenancy=False)"], + ["cell_control.transaction(conn, write)", "_read_rows(conn, count_tenancy=False)"], + ), + ("server/cli/cell_attach.py", "Run.run", ['cell_control.run("attach", self._owned)']), + ( + "server/cli/cell_attach_pg.py", + "_mode_try", + [ + "owner.call(lambda: _mode_transaction(owner.conn, lock_ms, mark, expected_cell, expected_fp), total_s=bound_s)" + ], + ), + ( + "server/cli/cell_control.py", + "Owner.transaction", + ["self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S)"], ), ( "server/cli/cell_attach.py", - "Run.run", - ["self.open_lock()", "cell_control.claim(self.lock_conn, self.router.call, self.row)"], + "Run._owned", + ["cell_control.claim(self.lock_conn, self.router.call, self.row)"], ), ( "server/cli/cell_attach.py", "Run.commit_mode", [ - "apg.commit_enforced(self.lock_conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" + "apg.commit_enforced(self.owner, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark, expected_cell=self.name, expected_fp=self.own_fp)" ], ), ( "server/cli/cell_attach_pg.py", - "_mode_try", + "_mode_transaction", ["cell_control.check_lock(conn)", "conn.transaction()", "tx.commit()"], ), ( @@ -354,7 +373,8 @@ def function(path: str, name: str) -> ast.AST: [ "claim(conn, call, row)", 'mutate(conn, call, row, "/limit", {"closed": True})', - "transaction(conn)", + "owner.transaction(clear_state)", + "owner.transaction(clear_triggers)", "paired(conn, row)", ], ), @@ -362,7 +382,7 @@ def function(path: str, name: str) -> ast.AST: "server/cli/cell.py", "_remove_locked", [ - 'cell_control.session("remove")', + 'cell_control.run("remove", remove)', "cell_control.claim(conn, call, row)", "cell_control.check_lock(conn)", ], @@ -483,7 +503,7 @@ def test_inventory_detects_new_literal_and_dynamic_writers(statement, expected): def test_acceptance_writer_callsites_are_classified(): # Another caller of a checked writer still needs its own claim and ownership proof. - names = {"_write_rows", "_mode_try", "commit_enforced", "backout"} + names = {"_write_rows", "_mode_try", "_mode_transaction", "commit_enforced", "backout"} actual = Counter() for path in (ROOT / "server").rglob("*.py"): found = inventory(path.read_text()) @@ -498,6 +518,7 @@ def test_acceptance_writer_callsites_are_classified(): { ("server/cli/cell_prepare.py", "_prepare_locked", "_write_rows"): 1, ("server/cli/cell_attach_pg.py", "commit_enforced", "_mode_try"): 1, + ("server/cli/cell_attach_pg.py", "_mode_try", "_mode_transaction"): 1, ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, } ) diff --git a/tests/unit/test_cell_prepare.py b/tests/unit/test_cell_prepare.py index 58040a584d..7c6e8f07fc 100644 --- a/tests/unit/test_cell_prepare.py +++ b/tests/unit/test_cell_prepare.py @@ -29,6 +29,7 @@ def _cell(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "storage_authority", "postgres") monkeypatch.setattr(settings, "pg_dsn", "postgresql://cell.internal/loom") monkeypatch.setattr(settings, "router_operator_token", OPERATOR) + # The attach lock's session is Postgres's (tests/integration/test_cell_attach.py). class Owner: conn = None diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 1471f5882b..226a1b54c0 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1102,8 +1102,17 @@ async def held(_conn: Any) -> None: pass monkeypatch.setattr(cell_control, "paired", paired) - monkeypatch.setattr(cell_control, "session", lambda _what: contextlib.nullcontext(object())) - monkeypatch.setattr(cell_control, "transaction", lambda _conn: contextlib.nullcontext()) + + class Owner: + conn = object() + + async def transaction(self, work): + return await work() + + async def run(_what, work): + return await work(Owner()) + + monkeypatch.setattr(cell_control, "run", run) monkeypatch.setattr(cell_control, "check_lock", held) monkeypatch.setattr(cell, "read_state_if_present", state) return checked From 789b25c15d9d17db191a74d53e50d25aece2d6aa Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:52:35 +0000 Subject: [PATCH 051/181] Refuse tagged YAML parser failures without quoting values Signed-off-by: zlareb1 --- server/cli/cell_update_config.py | 4 ++-- tests/unit/test_cell_update_config.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/cli/cell_update_config.py b/server/cli/cell_update_config.py index ce9e6452f2..a6303210db 100644 --- a/server/cli/cell_update_config.py +++ b/server/cli/cell_update_config.py @@ -32,8 +32,8 @@ def read_target(path: Path, current: Settings) -> Settings: """Overlay FILE on the CLI's effective settings; install neither the file nor values.""" try: values = read_settings_file(str(path), set(Settings.model_fields)) - except (SettingsFileError, UnicodeError, TypeError): - # Reader messages may quote a version, YAML source, or an arbitrary mapping key. + except Exception: # noqa: BLE001 — parser failures must not expose target file values. + # YAML constructors can raise builtin errors that contain their input. raise SettingsFileError( f"{path}: invalid target settings file; use version: 1, unique setting names " "and nonsecret values" diff --git a/tests/unit/test_cell_update_config.py b/tests/unit/test_cell_update_config.py index 8fbcb32444..28c3af0be8 100644 --- a/tests/unit/test_cell_update_config.py +++ b/tests/unit/test_cell_update_config.py @@ -145,6 +145,10 @@ def test_explicit_fingerprint_is_identical_to_the_process_and_uses_candidate_org "template", [ "version: 1\nsettings:\n openai_base_url: {marker}\n", + "version: 1\nsettings:\n tracker_org: !!int {marker}\n", + "version: 1\nsettings:\n tracker_org: !!float {marker}\n", + "version: 1\nsettings:\n tracker_org: !!bool {marker}\n", + "version: 1\nsettings:\n tracker_org: !!timestamp {marker}\n", "version: {marker}\nsettings: {{}}\n", "version: 1\nsettings:\n jwt_secret: [\n {marker}\n", "version: 1\nsettings:\n memory_ask_deep_evidence_max_memories: {marker}\n", From f9986b542078721e0e555e8bd861392f6b0e746b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:53:53 +0000 Subject: [PATCH 052/181] Propagate unexpected trigger errors before rollback cleanup Signed-off-by: zlareb1 --- server/core/cell_triggers.py | 5 ++- tests/integration/test_cell_attach_cleanup.py | 44 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 68e74777ec..784351d215 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -337,7 +337,10 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - except Exception: + except ( + asyncpg.exceptions.InsufficientPrivilegeError, + asyncpg.exceptions.LockNotAvailableError, + ): await tx.rollback() rolled_back = True raise diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 145be1188d..ca34ee7629 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -46,7 +46,7 @@ async def intercepted(self, query, *args, **kwargs): if query.startswith("SET LOCAL statement_timeout"): return await execute(self, "SET LOCAL statement_timeout = '10000ms'") if query == "RAISE Python error": - raise ValueError("trigger statement failed") + raise asyncpg.LockNotAvailableError("trigger statement busy") if phase == "rollback" and query.startswith("ROLLBACK"): await execute(self, "SELECT pg_sleep(10)") if owner == "mode" and query.startswith("LOCK TABLE"): @@ -109,3 +109,45 @@ async def dedicated(**_): if not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) + + +async def test_unexpected_trigger_error_reaches_owner_before_rollback(database, monkeypatch): + import asyncpg + + from server.core import cell + + conn, observer = database + pid = conn.get_server_pid() + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + execute = asyncpg.Connection.execute + original = ValueError("unexpected trigger operation failed") + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query == "RAISE unexpected Python error": + assert await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory')", + pid, + ) + raise original + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + started = time.monotonic() + with pytest.raises(ValueError) as error: + await apg.supervise( + conn, + cell_triggers._run_ddl( + conn, + "probe", + ["SELECT * FROM probe FOR UPDATE", "RAISE unexpected Python error"], + attempts=1, + warn=False, + ), + end=started + 0.5, + ) + assert error.value is original + assert time.monotonic() - started < 0.5 + await released(observer, pid) From 5a6207098742d4d1a2b87b3af59942eb6f4f070e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 23:53:53 +0000 Subject: [PATCH 053/181] Propagate unexpected trigger errors before rollback cleanup Signed-off-by: zlareb1 --- server/core/cell_triggers.py | 5 ++- tests/integration/test_cell_attach_cleanup.py | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/server/core/cell_triggers.py b/server/core/cell_triggers.py index 68e74777ec..784351d215 100644 --- a/server/core/cell_triggers.py +++ b/server/core/cell_triggers.py @@ -337,7 +337,10 @@ async def _run_ddl( await conn.execute(f"SET LOCAL lock_timeout = '{lock_timeout}'") for sql in sqls: await conn.execute(sql) - except Exception: + except ( + asyncpg.exceptions.InsufficientPrivilegeError, + asyncpg.exceptions.LockNotAvailableError, + ): await tx.rollback() rolled_back = True raise diff --git a/tests/integration/test_cell_attach_cleanup.py b/tests/integration/test_cell_attach_cleanup.py index 3b65c81b7d..77b3e297ce 100644 --- a/tests/integration/test_cell_attach_cleanup.py +++ b/tests/integration/test_cell_attach_cleanup.py @@ -45,7 +45,7 @@ async def intercepted(self, query, *args, **kwargs): if query.startswith("SET LOCAL statement_timeout"): return await execute(self, "SET LOCAL statement_timeout = '10000ms'") if query == "RAISE Python error": - raise ValueError("trigger statement failed") + raise asyncpg.LockNotAvailableError("trigger statement busy") if phase == "rollback" and query.startswith("ROLLBACK"): await execute(self, "SELECT pg_sleep(10)") if owner == "mode" and query.startswith("LOCK TABLE"): @@ -111,3 +111,44 @@ async def body(_owner): if not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) + + +async def test_unexpected_trigger_error_reaches_owner_before_rollback(database, monkeypatch): + import asyncpg + + from server.core import cell + + conn, observer = database + pid = conn.get_server_pid() + await conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + execute = asyncpg.Connection.execute + original = ValueError("unexpected trigger operation failed") + + async def intercepted(self, query, *args, **kwargs): + if self is conn: + if query == "RAISE unexpected Python error": + assert await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory')", + pid, + ) + raise original + if query.startswith("ROLLBACK"): + await execute(self, "SELECT pg_sleep(10)") + return await execute(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", intercepted) + started = time.monotonic() + with pytest.raises(ValueError) as error: + await _owner(conn).call( + lambda: cell_triggers._run_ddl( + conn, + "probe", + ["SELECT * FROM probe FOR UPDATE", "RAISE unexpected Python error"], + attempts=1, + warn=False, + ), + total_s=0.5, + ) + assert error.value is original + assert time.monotonic() - started < 0.5 + await released(observer, pid) From 5809472ea75dcb843bf774027ac8c7309f94adae Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:54:26 +0000 Subject: [PATCH 054/181] Retire participants only after permanent quiescence Keep request tails, actual application tasks, workers and native commands under one permanently closing process gate. Enroll before acceptance reads and retain uncertified retired rows until exact process death can be verified. Remove participant retire --yes and the retireLostPods chart value. Pod API object state and heartbeat expiry cannot authorize retirement or pruning. Signed-off-by: zlareb1 --- deploy/helm/loom/templates/_helpers.tpl | 5 +- .../helm/loom/templates/analytics-export.yaml | 2 +- deploy/helm/loom/templates/deployment.yaml | 2 +- deploy/helm/loom/templates/hooks.yaml | 4 +- deploy/helm/loom/templates/networkpolicy.yaml | 4 - deploy/helm/loom/templates/rbac.yaml | 32 - .../helm/loom/templates/serviceaccount.yaml | 2 +- deploy/helm/loom/templates/tests/health.yaml | 2 +- deploy/helm/loom/templates/workers.yaml | 2 +- deploy/helm/loom/values.yaml | 12 +- docs/architecture/cells.md | 50 +- docs/architecture/failure-and-recovery.md | 21 +- docs/architecture/provider-reconfiguration.md | 35 +- docs/operations/backup-and-recovery.md | 10 +- docs/operations/kubernetes.md | 26 +- docs/operations/replicas.md | 24 +- docs/reference/cli.md | 2 +- docs/reference/errors.md | 4 + server/auth/tenancy.py | 4 +- server/boot.py | 3 + server/cli/_ch.py | 40 +- server/cli/analytics_export.py | 17 +- server/cli/code_run.py | 3 +- server/cli/import_clickgap.py | 5 +- server/cli/import_recall.py | 13 +- server/cli/maintenance.py | 55 +- server/cli/participant.py | 27 +- server/cli/sql_access.py | 9 +- server/core/cell.py | 6 + server/core/instances.py | 3 + server/core/provider_process.py | 76 +- server/core/retirement.py | 114 +++ server/insights/app/provider_barrier.py | 82 +- server/insights/app/provider_command.py | 54 +- .../insights/store/provider_participants.py | 85 +- server/main.py | 130 ++- server/memory/app/embedder_calibration.py | 6 +- server/observability/events.py | 3 +- server/observability/problems.py | 5 +- server/transport/model_maintenance.py | 16 +- server/transport/retirement.py | 28 + server/worker_main.py | 18 +- server/workers/supervisor.py | 6 +- .../provider_process_before_retirement.py | 173 ++++ tests/integration/test_cell_retirement.py | 103 +++ .../test_provider_coordination_live_ch.py | 14 +- tests/unit/conftest.py | 9 + tests/unit/snapshots/cli_help.json | 6 +- tests/unit/snapshots/cli_tree.json | 20 +- tests/unit/test_analytics_export.py | 5 +- tests/unit/test_cell_cli_orgs.py | 9 +- tests/unit/test_cell_retirement.py | 752 ++++++++++++++++++ tests/unit/test_provider_barrier.py | 37 +- tests/unit/test_provider_command.py | 3 + tests/unit/test_provider_host_proof.py | 128 +-- tests/unit/test_reextract.py | 4 +- tests/unit/test_revocation_web_refresh.py | 30 +- tests/unit/test_shutdown_deadline.py | 2 +- 58 files changed, 1851 insertions(+), 491 deletions(-) delete mode 100644 deploy/helm/loom/templates/rbac.yaml create mode 100644 server/core/retirement.py create mode 100644 server/transport/retirement.py create mode 100644 tests/fixtures/provider_process_before_retirement.py create mode 100644 tests/integration/test_cell_retirement.py create mode 100644 tests/unit/test_cell_retirement.py diff --git a/deploy/helm/loom/templates/_helpers.tpl b/deploy/helm/loom/templates/_helpers.tpl index d0fee03c7a..fbbbdbac19 100644 --- a/deploy/helm/loom/templates/_helpers.tpl +++ b/deploy/helm/loom/templates/_helpers.tpl @@ -564,14 +564,11 @@ initContainers: {{/* Pod spec fields the Deployments, the hook Jobs and the analytics export share; `serviceAccount` names another account than the release's. */}} -{{/* `leads`: the pod may hold the workers lease, which alone reads the -Kubernetes API (retireLostPods). */}} {{- define "loom.podCommon" -}} -{{- $leads := .leads -}} {{- $sa := .serviceAccount -}} {{- with .root -}} serviceAccountName: {{ default (include "loom.serviceAccountName" .) $sa }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken (and .Values.retireLostPods $leads) }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} # Service links would inject LOOM_SERVICE_HOST and the like into Loom's env namespace. enableServiceLinks: false {{- if not .Values.sandbox.hostUsers }} diff --git a/deploy/helm/loom/templates/analytics-export.yaml b/deploy/helm/loom/templates/analytics-export.yaml index e6b0355650..403d7f0009 100644 --- a/deploy/helm/loom/templates/analytics-export.yaml +++ b/deploy/helm/loom/templates/analytics-export.yaml @@ -81,7 +81,7 @@ spec: app.kubernetes.io/component: analytics-export spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false "serviceAccount" $name) | nindent 10 }} + {{- include "loom.podCommon" (dict "root" $ "serviceAccount" $name) | nindent 10 }} {{- include "loom.waitForDevClickhouse" $ | nindent 10 }} containers: - name: analytics-export diff --git a/deploy/helm/loom/templates/deployment.yaml b/deploy/helm/loom/templates/deployment.yaml index 3ee7d1e98c..c53b91de59 100644 --- a/deploy/helm/loom/templates/deployment.yaml +++ b/deploy/helm/loom/templates/deployment.yaml @@ -73,7 +73,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" (not .Values.workers.enabled)) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} topologySpreadConstraints: - maxSkew: {{ .Values.topologySpread.zoneSkew }} diff --git a/deploy/helm/loom/templates/hooks.yaml b/deploy/helm/loom/templates/hooks.yaml index cdd65cccc1..d55dbe9a64 100644 --- a/deploy/helm/loom/templates/hooks.yaml +++ b/deploy/helm/loom/templates/hooks.yaml @@ -65,7 +65,7 @@ spec: app.kubernetes.io/component: {{ .name }} spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" $) | nindent 6 }} containers: - name: {{ .name }} image: {{ include "loom.image" $ }} @@ -83,7 +83,7 @@ spec: value: {{ $v | toString | quote }} {{- end }} # Its participant proof names the pod, so a hook pod lost mid-run is - # retired like any other (provider_process.pod_gone). + # verified against the same exact process identity as every participant. - name: POD_NAME valueFrom: { fieldRef: { fieldPath: metadata.name } } - name: POD_NAMESPACE diff --git a/deploy/helm/loom/templates/networkpolicy.yaml b/deploy/helm/loom/templates/networkpolicy.yaml index 6a1ed604cb..14903323dc 100644 --- a/deploy/helm/loom/templates/networkpolicy.yaml +++ b/deploy/helm/loom/templates/networkpolicy.yaml @@ -87,10 +87,6 @@ spec: {{- include "loom.anywhereBut" . | nindent 8 }} ports: - port: 443 - {{- if .Values.retireLostPods }} - # The Kubernetes API (443 on EKS; 6443 on most other clusters). - - port: 6443 - {{- end }} {{- if and (not .Values.devClickhouse.enabled) (ne (int .Values.clickhouse.port) 443) }} - port: {{ .Values.clickhouse.port }} {{- end }} diff --git a/deploy/helm/loom/templates/rbac.yaml b/deploy/helm/loom/templates/rbac.yaml deleted file mode 100644 index 72a0ff9d90..0000000000 --- a/deploy/helm/loom/templates/rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.retireLostPods }} -{{- if not (or .Values.serviceAccount.create .Values.serviceAccount.name) }} -{{- fail "retireLostPods grants `get pods` to the release's service account: set serviceAccount.create or serviceAccount.name, not the namespace's default" }} -{{- end }} -{{- /* Only `get` on pods: the workers leader asks whether a lost participant's -pod still exists (server/core/provider_process.py `pod_gone`). */}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: [pods] - verbs: [get] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "loom.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "loom.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/deploy/helm/loom/templates/serviceaccount.yaml b/deploy/helm/loom/templates/serviceaccount.yaml index 4536a11c5c..0c5c212fd8 100644 --- a/deploy/helm/loom/templates/serviceaccount.yaml +++ b/deploy/helm/loom/templates/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken .Values.retireLostPods }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} {{- end }} diff --git a/deploy/helm/loom/templates/tests/health.yaml b/deploy/helm/loom/templates/tests/health.yaml index 98a37832ac..ee21e05380 100644 --- a/deploy/helm/loom/templates/tests/health.yaml +++ b/deploy/helm/loom/templates/tests/health.yaml @@ -10,7 +10,7 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" . "leads" false) | nindent 2 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 2 }} containers: - name: health image: {{ include "loom.image" . }} diff --git a/deploy/helm/loom/templates/workers.yaml b/deploy/helm/loom/templates/workers.yaml index b89cce170b..710a38079c 100644 --- a/deploy/helm/loom/templates/workers.yaml +++ b/deploy/helm/loom/templates/workers.yaml @@ -42,7 +42,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" true) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} # Leader and standby on different nodes. topologySpreadConstraints: diff --git a/deploy/helm/loom/values.yaml b/deploy/helm/loom/values.yaml index 73c0ee9d30..5fe18416a7 100644 --- a/deploy/helm/loom/values.yaml +++ b/deploy/helm/loom/values.yaml @@ -205,15 +205,9 @@ serviceAccount: name: "" # IRSA (eks.amazonaws.com/role-arn) or other cloud identity annotations. annotations: {} - # Mount the service-account token (retireLostPods mounts it too). + # Mount the service-account token for workloads that explicitly need it. automountToken: false -# The workers leader retires the provider participant of a pod Kubernetes has -# deleted (lost with its node, evicted past its grace period), which would -# otherwise hold the next model change until `loom participant retire`. Grants -# the service account `get` on pods in this namespace and mounts its token. -retireLostPods: true - podAnnotations: {} podLabels: {} # amd64 by default: the C++ compiler tier's scip-clang ships for x86_64 only. The @@ -411,8 +405,8 @@ assistant: # Ingress to the Loom port from `ingressFrom` only. Egress to DNS, the # release's web pods, the extractor's pods (with extractor.enabled), the dev # ClickHouse (with devClickhouse.enabled), the EKS Pod Identity agent, and TCP -# 443, the ClickHouse port, postgres.port (with postgres.host) and 6443 (with -# retireLostPods) anywhere except `egressExcept` (providers, the IdP and git remotes +# 443, the ClickHouse port and postgres.port (with postgres.host) anywhere except +# `egressExcept` (providers, the IdP and git remotes # have no fixed range); a ClickHouse or Postgres inside the cluster needs an # extraEgress rule. Off, `guardMetadata` gives the Loom pods an egress-only policy # that allows every pod and every address but `egressExcept`. diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..9cd44ad6f4 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -18,6 +18,42 @@ that every transport enforces (`auth_actually_required()`, and `LOOM_MCP_AUTH_MO refuses to boot without them. [Storage authority](storage-authority.md) describes what a cell holds and how its two stores agree. +## Participant lifecycle baseline + +A participant advertising the ninth `host_proof` field `cell-retirement-v1` +enrolls before its first fresh primary acceptance read and uses permanent guarded +retirement. A serving cell heartbeat includes its `participant_id` beside the +fingerprint, allowing a replacement to be joined to the participant registry. +Workers children enroll independently and do not publish serving instance rows. +The capability identifies installed code; it does not prove that every older +process or unsafe retirement client has disappeared. + +Before online configuration finalization is supported, stop every unsafe manual +retirement and pruning writer and prevent its binary/image from restarting. +Inventory and positively terminate every accepting server, workers child, +standalone worker and native command that overlapped those writers. Include paused +boots, already retired/pruned identities and lifecycle-capable processes started +during the overlap: old maintenance code could have hidden their rows too. Only +then start replacement accepting and maintenance processes with this lifecycle. +A shared image tag, pod deletion, readiness or heartbeat expiry is insufficient. +Keep exact PID/boot/namespace/start or trusted runtime termination evidence with +the deployment's upgrade record. No CLI flag or stored acknowledgement substitutes +for that evidence, and the runtime census does not verify the external inventory. + +Census retained legacy rows while unsafe writers are stopped. Recertify a retained +identity only after exact verifiable physical death and known-zero cross-replica +queries. Keep empty or unknown proofs quarantined. If the legacy inventory is +incomplete, stop all accepting/native-command and retirement/pruning processes, +positively terminate or fence every runtime that could execute an old binary, and +prevent old restarts. Only after that physical quiescence may the operator request +an exact audited cleanup of selected unknown legacy participant identities. There +is no automatic age-based cleanup or affirmation bypass. Restart only guarded +lifecycle processes afterward. Unknown or partitioned runtimes remain blockers. + +This baseline is an operational prerequisite, not a release-cut requirement. +Configuration finalization also needs the directory's controller-fencing protocol; +the lifecycle certificate alone cannot authorize reservation release. + ## Why whole deployments A cell shares no store, key, lease or worker with another. Every guarantee a single @@ -485,11 +521,13 @@ with the same configuration, or `--finish`, first writes `next` wherever it is m It reruns attach's overlap refusals against every other cell's directory row, counting both values of each, writes the new fingerprint as `next` to the directory and then to `loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. `loom cell update --finish` waits, at most `--wait` (default 10 minutes), until -every live process reports `next` and the longest request deadline has passed; when the -change turns SCIM on or changes the cell's OIDC client it then runs attach's SCIM -precondition (refusing, or `--revoke-unprovisioned`); only then does it make `next` the -only value, in `loom.cell_self` first and the directory last. A widened issuer or +out. Finalization first narrows the cell's accepted value durably while the directory +keeps both reservations. It then requires a complete post-cutoff participant census, +permanent guarded retirement or verified exact death for every captured identity, +and a serving target replacement outside that captured set. Heartbeat expiry and +request deadlines do not prove this. Only afterward may the directory release its +old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) +and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or pattern is reserved in the directory before any process accepts it, and a narrowed one is released only after none does. A refusal or a wait that runs out keeps both values and prints the processes not yet reporting `next`; rerunning resumes, and @@ -514,6 +552,8 @@ version and fields, and beside them keys that no fingerprint compares: they are needs no column for them (`server.core.instances.cell_row`). Their presence proves the build that reports them; there is no generation key. +- `participant_id`: this server's provider participant UUID, for joining its actual + process lifecycle to the serving instance row. A workers child publishes no instance row. - `serves`: `LOOM_SERVES_TRAFFIC`. - `marked`: whether this process's pooled Postgres sessions keep the tenancy marker (`loom.tenancy_checked`). The cell refresh reads it with the state rows, after `RESET ALL` diff --git a/docs/architecture/failure-and-recovery.md b/docs/architecture/failure-and-recovery.md index 40f8e4b57e..d54fb025b8 100644 --- a/docs/architecture/failure-and-recovery.md +++ b/docs/architecture/failure-and-recovery.md @@ -186,17 +186,16 @@ node. Nothing else counts: an unreachable process is not a stopped one. TTL is 60 s, renewed every 20 s; a contender takes it over once it expired and the holder's participant counts as stopped. `loom provider-lock clear --holder ` frees one whose holder has no participant row, which never counts as stopped. -- **Killed mid-reconfiguration.** The transition waits on a participant whose heartbeat - expired, since a paused process may resume with admitted work, until a process on - its host proves it gone (its container restarted, or its pid exited) and retires it. - The workers leader - deletes the rows of participants retired a day ago and lists, once a minute, those a - transition would wait on as `provider_participants_stale` in its `/v1/health`. Retire - one by hand with `loom participant retire --participant --yes` once its process - is confirmed gone; the command refuses a participant whose heartbeat is fresh or whose - queries still run. -- **A boot that fails after enrolling** retires its row; a crash-looping container leaves - none behind. +- **Killed mid-reconfiguration.** An expired heartbeat remains a blocker until the + exact process dies or completes guarded shutdown. A trusted local process can + verify its PID, namespace, boot and start identity, then retire it after all + cross-replica queries settle. Use `loom participant retire --participant ` on + that trusted host. Pod deletion or replacement and namespace changes alone do + not prove death. The workers leader prunes only certified permanent retirements + older than a day; unknown legacy rows remain visible. +- **A boot that fails after enrolling** remains nonretired unless all its actual + producers and writes stop. A shutdown deadline does not turn cancellation into + a retirement certificate. A process frozen past its heartbeat (a stopped container, a VM pause) holds the transition until it resumes and drains, or an operator retires it. diff --git a/docs/architecture/provider-reconfiguration.md b/docs/architecture/provider-reconfiguration.md index 71fa716069..622fb99dc4 100644 --- a/docs/architecture/provider-reconfiguration.md +++ b/docs/architecture/provider-reconfiguration.md @@ -36,21 +36,38 @@ stopped only once it retired itself (a graceful shutdown), a later process on it host proved it gone, or an operator retired it. An expired heartbeat is no proof: a paused process can hold admitted work outside ClickHouse and write it with the old configuration once it resumes, so the transition waits for it to beat again and -drain. Each row carries its host name, kernel boot, container id (Docker's, or a -Kubernetes pod's and container name), pid namespace, pid and start time; every 10 s -each process retires the participants of its own host and boot whose pid no longer -runs that process, or that ran in its own container before the container restarted -into a new pid namespace, once no query of theirs runs. That covers a killed process -whose container or worker restarts in place; two containers that share a host name -(host networking, two containers of one pod) never retire each other. An operator who confirmed its -process is gone retires it with `loom participant retire --participant --yes`, -which refuses one whose heartbeat is fresh or whose queries still run. The workers +drain. Each row carries its host name, kernel boot, container identity, pid namespace, +pid and start time. A later process on the same trusted host and pid namespace can +prove that the exact PID exited or was reused. A changed namespace, a missing pod +object, another pod UID or a terminal API phase does not prove physical termination. +`loom participant retire --participant ` performs that exact local verification +and refuses when evidence is unavailable or any cross-replica query remains. +The workers leader's `/v1/health` lists every participant a transition would wait on as `provider_participants_stale` (`heartbeat_expired` or `queries_running`). `system.processes` is read over `clusterAllReplicas(LOOM_CH_CLUSTER)`; a server without that cluster answers for itself, and any other failure of the read counts the participant as running. +Permanent retirement uses a separate process gate that never reopens. The outer +HTTP/WebSocket scope covers request-tail settlement; an inner scope covers actual +application tasks across middleware child-task boundaries, including OAuth and +control handlers. Worker passes and native command bodies use the same counter. +Shutdown closes admission, stops producers, waits for actual scope and registered +descendant completion, flushes buffers and checks carried and server-side writes. +Canceling a coroutine or exhausting the shutdown budget leaves its row nonretired. +A failed startup likewise cannot certify unfinished producers. Model-transition +`drained` is reversible and does not certify permanent retirement. + +The ninth `host_proof` field, `cell-retirement-v1`, identifies this lifecycle. +Only the guarded shutdown or verified-death writer publishes certified retirement. +The generic heartbeat writer cannot retire or reopen a retired identity; a +replacement process draws a fresh UUID. Pruning checks the latest certified row +before deleting all versions. Uncertified legacy retired rows stay visible and +cannot disappear through an age-based prune. Old reapers fail closed on the ninth +field, but old manual/pruning binaries must be stopped before the +[cells lifecycle baseline](cells.md#participant-lifecycle-baseline) is eligible. + Each bounded rebuild step holds the shared configuration Keeper key, renewed every 20 s (TTL 60 s). A contender takes over an expired key only when its holder's participant counts as stopped; an elapsed TTL alone is no proof. `loom diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 02de0a5c91..b2dfa6a12c 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -504,11 +504,11 @@ Bounded exceptions: - The events stream (budget violations) is drained on `SIGTERM`. A `SIGKILL` or OOM loses at most one flush interval of that telemetry, never audit or memory rows. -- Provider reconfiguration waits on a dead process until a process on its host proves it - gone, or, on Kubernetes, until the workers leader sees its pod gone. Elsewhere, one - whose host is gone too (a move to a new host) is retired by hand with - `loom participant retire --participant --yes` - ([provider reconfiguration](../architecture/provider-reconfiguration.md)). +- Provider reconfiguration waits on a process until guarded retirement or exact + physical-death proof with settled queries. On its trusted host, run + `loom participant retire --participant `. Pod deletion and heartbeat expiry + are insufficient. If its identity is unknown, keep the row until the + [physically quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). - Rate-limit bucket fill is per host. It refills in seconds, so a new host starts with full buckets. diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index 8eb2749050..3e31ef253e 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -189,17 +189,19 @@ graceful stop, and within the 30 s lease TTL when a pod or node dies. Every pod labelled `deploy:k8s:`, so a new leader runs the workers whose last pass is overdue rather than waiting out their interval again. -A pod that dies without stopping (node loss, eviction past the grace period) leaves its -provider participant row, which would hold the next embedding or model change from the -dashboard. With `retireLostPods` (the default) the workers leader retires it once its -heartbeat expired, no query of it runs and Kubernetes says the pod is gone: its object -deleted or replaced, or the pod `Failed` (evicted). That is a minute or two after a -deletion or an eviction, and after a node loss once Kubernetes deletes the node's pods -(five minutes of `NotReady` by default). It needs `get` on pods in the namespace (a Role -the chart creates for the release's service account) and that account's token, which only -the pods that may hold the workers lease mount. Without it, `/v1/health` names the row; -once the pod is gone, retire it with -`loom participant retire --participant --yes` ([replicas](./replicas.md#stopping-a-replica)). +A pod that dies without stopping leaves its provider participant row. A missing +or replaced pod object, a Failed/Succeeded API phase and a completed rollout do +not prove that an old process on a partitioned node stopped. Such rows remain +blockers until guarded shutdown or verified exact physical death with settled +queries. Loom requests no pod-read permission or service-account token for +participant retirement. The explicit `serviceAccount.automountToken` value stays +available for workloads that need a token for another integration. + +Use `loom participant retire --participant ` on the trusted host where its +exact PID/namespace/start identity is verifiable. If that evidence is unavailable, +use the [quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline) +after positively terminating or fencing every relevant runtime. API object +cleanup alone cannot authorize retirement. There is no migration Job. Every pod migrates at boot under a ClickHouse lease, so a Job would only repeat that step, and a pre-upgrade hook would run the new schema while @@ -388,7 +390,7 @@ NetworkPolicy whose one rule allows every pod and every address but those (none release: Helm refuses to take over an object it does not own, so the upgrade fails. - An engine that keeps address ranges off node IPs (Cilium's default) blocks host-network endpoints and the nodes' own addresses: NodeLocal DNSCache (so DNS), the EKS Pod - Identity agent, and the API server (which `retireLostPods` calls) where the engine + Identity agent, where the engine gives it an identity of its own. Turn on the engine's node matching, or set `guardMetadata: false` and allow them in your own policy. diff --git a/docs/operations/replicas.md b/docs/operations/replicas.md index f4c8729ef4..e635c0a490 100644 --- a/docs/operations/replicas.md +++ b/docs/operations/replicas.md @@ -101,14 +101,22 @@ it. ## Stopping a replica -Stop it gracefully (`docker stop`, SIGTERM): it marks its heartbeat row stopped -and retires its provider participant row. A forced kill leaves both; the -heartbeat row stops counting after three intervals. A process that restarts in -the same container retires the dead one's participant row itself. If a provider -reconfiguration waits on a dead process's row from elsewhere, retire it with the -`participant_id` its `/v1/health` reported, once you have confirmed it is gone -(the command refuses one whose heartbeat is fresh or whose queries still run): +Stop it gracefully (`docker stop`, SIGTERM). It first closes admission permanently, +stops its producers, and waits for actual request, worker and registered descendant +completion. It retires only after buffered, carried and server-side writes settle. +A canceled task that keeps executing or an exhausted shutdown budget leaves the +participant nonretired. A stopped or expired instance heartbeat is diagnostic. + +A trusted process on the same host and pid namespace can prove the exact old PID +exited or was reused, then retire it after all cross-replica queries settle: ```bash -loom participant retire --participant --yes +loom participant retire --participant ``` + +The command verifies retained boot, PID namespace and process-start identity; it +refuses remote or unknown identity evidence and live/uncountable queries. A +changed container namespace or Kubernetes API object is insufficient. An operator +assertion does not override these checks. For an inaccessible host, positively +terminate or fence the exact runtime before the documented +[quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..3e0b9068ea 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -266,7 +266,7 @@ python scripts/codegen/docs_reference.py | `loom down` | Stop the local stack while preserving data; --delete-data performs a confirmed reset. | `--root`, `--file`, `--env-file`, `--delete-data`, `-y` | | `loom ch-setup` | Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. | `--print-sql`, `--print-analyst-sql` | | `loom participant` | The Loom processes a provider change waits for. | | -| `loom participant retire` | Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health). | `--participant`, `--yes` | +| `loom participant retire` | Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health). | `--participant` | | `loom deployment` | The configuration every replica checks itself against. | | | `loom deployment publish` | Publish this env's configuration as the deployment's, replacing its root secret or LOOM_EMBEDDING_MODEL/DIM on purpose. | `--yes`, `--check` | | `loom settings` | Every setting's value in force, and overrides of the live ones. | | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 62a4a86f46..792a3cd4ba 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -752,6 +752,10 @@ Raised in `server/router/app.py`. Category `capacity`, retryable: yes. Raised in `server/router/app.py`. Category `dependency`, retryable: yes. +#### `PROCESS_STOPPING` + +Raised in `server/transport/model_maintenance.py`, `server/transport/retirement.py`. + #### `PROJECTION_BEHIND` HTTP 503, raised as `server.memory.app.pg_admission.ProjectionBehindError`. 503 — earlier commits in this namespace are not yet in ClickHouse, so a read of ClickHouse that must see every write the namespace committed would miss them. Category `capacity`, retryable: yes for an idempotent op, no for another op once it ran. diff --git a/server/auth/tenancy.py b/server/auth/tenancy.py index 8c6b4c9437..5042cb22b5 100644 --- a/server/auth/tenancy.py +++ b/server/auth/tenancy.py @@ -262,7 +262,9 @@ async def _place_once(org: str) -> tuple[str, str]: request cancelled leaves the call running for the others.""" task = _placing.get(org) if task is None: - task = asyncio.create_task(_place(org)) + from server.core.background import create_deferred_task + + task = create_deferred_task(_place(org), name="tenancy-placement") _placing[org] = task task.add_done_callback(functools.partial(_placed, org)) return await asyncio.shield(task) diff --git a/server/boot.py b/server/boot.py index 69029d5b34..90ba996694 100644 --- a/server/boot.py +++ b/server/boot.py @@ -463,6 +463,9 @@ def _spawn_boot_loop(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Ta """A task that runs for the process's life: cancelled at shutdown, never registered as deferred, since a model transition's drain waits for those.""" task = asyncio.get_running_loop().create_task(coro, name=name) + from server.core.retirement import get_gate + + get_gate().track(task) _BOOT_TASKS.add(task) task.add_done_callback(_BOOT_TASKS.discard) return task diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 042d8d1de0..184aa1f6eb 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -295,10 +295,7 @@ async def connect_declared( from server.core.pg_profile import check_configured await check_configured(ch) - if org_free is not None and not named: - await check_org_free(org_free) - else: - await check_orgs(named) + await check_acceptance(orgs=named, org_free=org_free) except BaseException: await disconnect(ch) raise @@ -315,6 +312,38 @@ async def connect_code_plane(ch: CHClient, args: argparse.Namespace) -> None: await connect_declared(ch, orgs=() if getattr(args, "dry_run", False) else (org,)) +async def check_acceptance( + *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None +) -> None: + """Fresh post-enrollment gate check on already verified, open paired stores.""" + named = _declared(orgs, org_free) + if org_free is not None and not named: + await check_org_free(org_free) + else: + await check_orgs(named) + + +async def connect_verified(ch: CHClient) -> None: + """Verify paired stores without obtaining an acceptance view or admitting work. + + Native commands enroll next, then repeat their accepting org/cell checks. + Operator-only commands may keep using the org-free checked composition. + """ + from server.core.pg_profile import check_configured + + if not ch.connected: + await ch.connect() + try: + await check_configured(ch) + except BaseException: + await disconnect(ch) + raise + from server import boot + from server.core.background import register_buffer_drainer + + register_buffer_drainer("boot-writes", boot.drain_writes) + + async def connect_checked( ch: CHClient, *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None ) -> None: @@ -354,9 +383,10 @@ def run_connected[T]( async def _go() -> T: ch = get_client() - await connect_checked(ch, orgs=named, org_free=org_free) + await connect_verified(ch) try: async with model_command(): + await check_acceptance(orgs=named, org_free=org_free) return await run(ch) finally: await disconnect(ch) diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index f69e3f9396..5f5abf8bb3 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -9,7 +9,7 @@ from datetime import date, timedelta from typing import Any -from server.cli._ch import connect_declared, disconnect, reads +from server.cli._ch import check_acceptance, connect_verified, disconnect, reads from server.core.datetimes import utc_now as _now # The first botocore whose put_object takes IfNoneMatch (the `aws` extra's floor). @@ -90,15 +90,20 @@ def new_client() -> Any: check_credentials(role_arn=args.role_arn, region=args.region) put = s3_put(new_client, args.bucket, expected_owner=args.expected_bucket_owner) - # No model calls and no writes to ClickHouse: a plain connection, outside - # the provider barrier that `run_connected` enters. It connects inside the - # deadline's scope, so the driver's read timeout is the deadline too. + # Analytics export consumes org data even without inference; its entire native + # body participates in permanent retirement without taking model admission. + from server.insights.app.provider_command import participant_command + async def _go() -> list[Exported]: with statement_timeout(STATEMENT_TIMEOUT_S): ch = get_client() - await connect_declared(ch, org_free=reads("reads; writes only the export bucket")) + await connect_verified(ch) try: - return await export_day(day, prefix=args.prefix, put=put, name_key=name_key, ch=ch) + async with participant_command(): + await check_acceptance(org_free=reads("reads; writes only the export bucket")) + return await export_day( + day, prefix=args.prefix, put=put, name_key=name_key, ch=ch + ) finally: await disconnect(ch) diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 5479c3d851..1074951fa1 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -117,10 +117,11 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) try: # A repository added from the Repositories page lives in the registry, not in # LOOM_CODE_REPOS_JSON: read it before the subcommand resolves its namespace. + from server.cli._ch import connect_verified from server.code.app.index.repo_registry import REGISTRY_UNREAD_MESSAGE, refresh_db_repos from server.code.domain.index.repos import RegistryUnread - await connect_declared(get_client(), org_free=reads("reads the repository registry")) + await connect_verified(get_client()) async with model_command(): # Never raises; while it has never succeeded, `configured_repos` refuses. await refresh_db_repos(force=True) diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 60f3fdc6fc..983ce671cd 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -716,15 +716,16 @@ async def _clickgap_write_category_links(w: _ClickgapWrite) -> None: async def _clickgap_enroll(stack: Any, org: str) -> Any: """Connect and enroll before selecting the importer's effective embedder.""" from server import boot - from server.cli._ch import connect_checked + from server.cli._ch import check_acceptance, connect_verified from server.core.background import register_buffer_drainer from server.core.ch import get_client from server.embedding import get_embedder from server.insights.app.provider_command import model_command - await connect_checked(get_client(), orgs=(org,)) + await connect_verified(get_client()) register_buffer_drainer("boot-writes", boot.drain_writes) await stack.enter_async_context(model_command()) + await check_acceptance(orgs=(org,)) return get_embedder() diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index fd0925accb..b24c9371b5 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -781,22 +781,29 @@ async def _http_writer(op: str, payload: dict[str, Any]) -> None: return _http_writer, client.aclose - from server.cli._ch import connect_checked, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client handlers = inproc_ops() ch = get_client() - await connect_checked(ch, orgs=(args.org,)) + await connect_verified(ch) from server import boot from server.core.background import register_buffer_drainer from server.insights.app.provider_command import model_command register_buffer_drainer("boot-writes", boot.drain_writes) command = model_command() + entered = False try: await command.__aenter__() + entered = True + await check_acceptance(orgs=(args.org,)) except BaseException: - await disconnect(ch) + try: + if entered: + await command.__aexit__(None, None, None) + finally: + await disconnect(ch) raise async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 53cb8e4004..9a6cdb2437 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -207,8 +207,9 @@ def _cmd_reextract(args: argparse.Namespace) -> int: the stored transcripts, the source untouched.""" import asyncio - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command from server.memory.app.reextract import load_sessions, render_report, replay if args.into == args.namespace: @@ -225,33 +226,37 @@ def _cmd_reextract(args: argparse.Namespace) -> int: async def _go() -> int: ch = get_client() - await connect_declared( - ch, org_free=reads("reads; the replay writes through the server at --base-url") - ) + await connect_verified(ch) try: - sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + async with participant_command(): + await check_acceptance( + org_free=reads("reads; replay writes through the HTTP server") + ) + sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + if args.limit_sessions: + sessions = sessions[: args.limit_sessions] + if not sessions: + print( + f" no set_from_messages evidence in {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + rep = await replay( + sessions, + base_url=args.base_url, + token=token, + org=args.org, + into=args.into, + dates=args.dates, + concurrency=args.concurrency, + dry_run=args.dry_run, + ) + print( + render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run) + ) + return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: await disconnect(ch) - if args.limit_sessions: - sessions = sessions[: args.limit_sessions] - if not sessions: - print( - f" no set_from_messages evidence in {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - rep = await replay( - sessions, - base_url=args.base_url, - token=token, - org=args.org, - into=args.into, - dates=args.dates, - concurrency=args.concurrency, - dry_run=args.dry_run, - ) - print(render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run)) - return 1 if rep.failed and rep.failed == rep.sessions else 0 return asyncio.run(_go()) diff --git a/server/cli/participant.py b/server/cli/participant.py index e42832b6af..9154bf4be4 100644 --- a/server/cli/participant.py +++ b/server/cli/participant.py @@ -1,10 +1,9 @@ """`loom participant retire`: the manual recovery for a Loom process that died without shutting down. -Every Loom process registers a provider participant row, and a provider -reconfiguration waits until each live participant acknowledges the drain. A -participant that stops beating expires by itself; this retires one at once, -once an operator has confirmed its process is gone. +A participant retires only after permanent guarded shutdown or verifiable exact +physical death on a trusted host, with its cross-replica queries settled. +Heartbeat age and an operator affirmation are not evidence of death. """ from __future__ import annotations @@ -17,8 +16,7 @@ def _cmd_retire(args: argparse.Namespace) -> int: - """Retire a provider participant row whose process died without shutting - down, so a provider reconfiguration need not wait for it to expire.""" + """Verify exact process death and retire the participant after its queries settle.""" from uuid import UUID from server.cli._ch import run_connected @@ -28,13 +26,6 @@ def _cmd_retire(args: argparse.Namespace) -> int: except ValueError: print(f" ✗ {args.participant!r} is not a participant id", file=sys.stderr) return 2 - if not args.yes: - print( - " ✗ retiring a participant that is still running lets a reconfiguration proceed " - "under its writes; confirm its process is stopped, then pass --yes", - file=sys.stderr, - ) - return 2 async def _go(ch: Any) -> str: from server.insights.app.provider_barrier import retire_participant @@ -47,8 +38,9 @@ async def _go(ch: Any) -> str: return 1 if outcome == "live": print( - f" ✗ participant {participant} is still running: its heartbeat is fresh or " - "a query of its own still runs", + f" ✗ participant {participant} has no verifiable exact physical-death proof " + "or its cross-replica queries have not settled; run on its trusted host after " + "the exact process stops. Expiry and Kubernetes object deletion are not proof", file=sys.stderr, ) return 1 @@ -61,9 +53,8 @@ def register_participant_cli(sub: Any) -> None: ssub = p.add_subparsers(dest="participant_cmd", required=True) r = ssub.add_parser( "retire", - help="Retire the provider participant of a process that died without shutting " - "down (its id is `participant_id` in that process's /v1/health).", + help="Verify exact physical death on this trusted host and retire its participant " + "after all queries settle (the id is participant_id in /v1/health).", ) r.add_argument("--participant", required=True, help="The participant id to retire") - r.add_argument("--yes", action="store_true", help="Its process is confirmed stopped") r.set_defaults(func=_cmd_retire) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 8499eed5cd..6a727c9e7b 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -70,14 +70,17 @@ def _connected( ) -> Any: """Run `work(ch)` on one plain connection: the command calls no model. `orgs` or `org_free` as `server.cli._ch.connect_declared` takes them.""" - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command async def _go() -> Any: ch = get_client() - await connect_declared(ch, orgs=orgs, org_free=org_free) + await connect_verified(ch) try: - return await work(ch) + async with participant_command(): + await check_acceptance(orgs=orgs, org_free=org_free) + return await work(ch) finally: await disconnect(ch) diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..a37ce9e029 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -272,6 +272,12 @@ def state(self) -> str: _awaiting_first_read = False +def discard_inherited_view() -> None: + """A child enrolls and reads primary state before using acceptance state.""" + global _view, _memo, _awaiting_first_read + _view, _memo, _awaiting_first_read = _View(), None, True + + def expect_first_read() -> None: global _awaiting_first_read _awaiting_first_read = True diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..ecbe666c07 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -142,9 +142,12 @@ def cell_row() -> dict[str, Any]: `marked` (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" + from server.core.provider_process import process_identity + reported = { **cell.as_value(cell.mine()), "serves": bool(settings.serves_traffic), + "participant_id": str(process_identity()), "marked": cell.marked(), **cell_router.router_key(), } diff --git a/server/core/provider_process.py b/server/core/provider_process.py index 6ef0d6af5a..a2c6ad6328 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -5,11 +5,9 @@ still has a query running. """ -import asyncio import os import re import socket -from pathlib import Path from uuid import UUID, uuid4 _identity: tuple[int, UUID] | None = None @@ -99,11 +97,14 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace), or it ran in this container - before the container restarted into a new pid namespace (which kills every - process of the old one). Two containers that share a host name never prove - each other stopped; a process that cannot be inspected is not proven stopped.""" + the one that started then (same pid namespace). Another namespace or API + object proves nothing. Sharing a host name or container identity alone does + not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") + if len(parts) == 9: + if parts[8] != LIFECYCLE: + return False + parts = parts[:8] if len(parts) not in (6, 7, 8): return False host, boot, container, pidns, pid, start = parts[:6] @@ -114,7 +115,7 @@ def proves_stopped(proof: str) -> bool: if host != socket.gethostname() or boot != my_boot: return False if pidns != my_pidns: - return bool(container) and container == _container() + return False # namespace identity alone says nothing about old processes # Only positive evidence: no such process, or another one under its pid. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: @@ -129,45 +130,36 @@ def proves_stopped(proof: str) -> bool: return False -_SA = "/var/run/secrets/kubernetes.io/serviceaccount" -_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") +LIFECYCLE = "cell-retirement-v1" -async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: - """Whether Kubernetes says the pod `proof` names is gone: no pod of that name - in its namespace, one with another UID, or this one in phase Failed or - Succeeded (an evicted pod keeps its object; a pod that restarts its - containers reaches those phases only once every one stopped for good). The - kubelet removes a pod's object once its containers stopped (or an operator - force-deleted it), so each is positive evidence a lost pod's process ended. - Asked only of this pod's own namespace, with its service-account token; - False wherever the API or the proof does not say.""" +def certified(proof: str) -> str: + """Preserve the exact captured identity and stamp the installed protocol.""" parts = proof.split("|") - found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None - namespace = os.environ.get("POD_NAMESPACE", "") - if not found or not namespace or parts[6] != namespace or not parts[7]: + if len(parts) == 9 and parts[8] == LIFECYCLE: + parts = parts[:8] + if len(parts) not in (6, 7, 8): + raise ValueError("Unrecognized participant identity proof") + parts += [""] * (8 - len(parts)) + if not all(parts[i] for i in (0, 1, 3, 4, 5)): + raise ValueError("Incomplete participant identity proof") + if not parts[4].isdecimal() or int(parts[4]) <= 0 or not parts[5].isdecimal(): + raise ValueError("Invalid participant process identity") + return "|".join((*parts, LIFECYCLE)) + + +def has_certificate(proof: str) -> bool: + if len(proof.split("|")) != 9: return False - host = os.environ.get("KUBERNETES_SERVICE_HOST", "") - port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") try: - token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() - except OSError: - return False - if not host or not token: + return certified(proof) == proof + except ValueError: return False - import httpx - url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" - try: - async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: - r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) - except (httpx.HTTPError, OSError): - return False - if r.status_code == 404: - return True - if r.status_code != 200: - return False - pod = r.json() - if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): - return True - return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") + +def participant_proof() -> str: + """Advertise capability only in a composition with installed lifecycle guards.""" + from server.core.retirement import get_gate + + proof = host_proof() + return certified(proof) if proof and get_gate().role else proof diff --git a/server/core/retirement.py b/server/core/retirement.py new file mode 100644 index 0000000000..025a437362 --- /dev/null +++ b/server/core/retirement.py @@ -0,0 +1,114 @@ +"""Permanent process admission and the evidence passed to the retirement writer. + +Model maintenance can reopen. This gate never does: the participant identity is +finished once shutdown begins, even when its shutdown budget expires. +""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from server.core import provider_process + + +class ProcessStopping(RuntimeError): + """This participant can never accept another unit of work.""" + + +class RetirementGate: + """One event loop; checking admission and incrementing never await.""" + + def __init__(self) -> None: + self.active = 0 + self.closed = False + self.role = "" + self.enrolled = False + self.producers_stopped = False + self.producers: set[asyncio.Task[Any]] = set() + + def install(self, role: str) -> None: + if self.closed or self.role not in ("", role): + raise ProcessStopping("A retired participant cannot enroll again") + self.role = role + + def close(self) -> None: + self.closed = True + + def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: + """A process-lifetime producer must actually end, not merely be canceled.""" + self.producers.add(task) + task.add_done_callback(self.producers.discard) + if self.closed: + task.cancel() + raise ProcessStopping("Cannot start a producer during shutdown") + return task + + def stopped(self) -> None: + """Composition calls this after stopping every role-specific producer.""" + self.close() + self.producers_stopped = True + + @property + def quiescent(self) -> bool: + return ( + self.closed + and bool(self.role) + and self.producers_stopped + and self.active == 0 + and all(task.done() for task in self.producers) + ) + + @asynccontextmanager + async def admit(self) -> AsyncIterator[None]: + if self.closed or (self.role and not self.enrolled): + raise ProcessStopping("This process is stopping or has not enrolled") + self.active += 1 + try: + yield + finally: + self.active -= 1 + + +_pid = os.getpid() +_gate = RetirementGate() + + +def get_gate() -> RetirementGate: + """A fork cannot inherit an accepting view or its parent's retirement state.""" + global _pid, _gate + if _pid != os.getpid(): + role = _gate.role + _pid, _gate = os.getpid(), RetirementGate() + _gate.role = role + return _gate + + +@dataclass(frozen=True) +class RetirementProof: + """Internal evidence; only guarded shutdown or verified death constructs it.""" + + participant: UUID + host_proof: str + after: int + version: int | None = None + + +def guarded_proof(participant: UUID, *, version: int) -> RetirementProof: + gate = get_gate() + proof = provider_process.host_proof() + if not gate.quiescent or not gate.enrolled or not proof: + raise ProcessStopping("Permanent guarded quiescence is not established") + return RetirementProof(participant, provider_process.certified(proof), version - 1, version) + + +def death_proof(participant: UUID, proof: str, *, after: int) -> RetirementProof | None: + """Called only after the complete cross-replica running-query census is zero.""" + if not provider_process.proves_stopped(proof): + return None + return RetirementProof(participant, provider_process.certified(proof), after) diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 503cd021e2..193fcc9335 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -11,6 +11,7 @@ import asyncio import contextlib import logging +import os import threading from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager @@ -18,10 +19,11 @@ from typing import Any from uuid import UUID +from server.core import retirement from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import query_tag from server.core.model_work_gate import get_model_work_gate -from server.core.provider_process import host_proof, pod_gone, process_identity, proves_stopped +from server.core.provider_process import participant_proof, process_identity, proves_stopped from server.insights.store.provider_participants import ( OWN_VERSION_SQL, SEQUENTIAL, @@ -30,6 +32,7 @@ participants, prune_retired, record_participant, + record_retirement, running_queries, ) @@ -62,6 +65,8 @@ def stamp( ) -> tuple[UUID, str, int]: with self.lock: if transition is not None and state is not None: + if self.state == "retired" and state != "retired": + raise retirement.ProcessStopping("A retired UUID cannot re-enroll") self.transition, self.state = transition, state self.version = max(time_ns(), self.version + 1, floor + 1) return self.transition, self.state, self.version @@ -109,7 +114,7 @@ def beat(self) -> None: transition, state, version = _OWN.stamp(floor=seen) if state == "retired": return - row = [me, transition, state, version, STALE_AFTER_S, host_proof()] + row = [me, transition, state, version, STALE_AFTER_S, participant_proof()] self.ch.insert(TABLE, [row], column_names=_COLUMNS, settings=_TAG) def _run(self) -> None: @@ -146,6 +151,7 @@ def signal_stop(self) -> None: _HEARTBEAT = _Heartbeat() _registered = False +_process_pid = os.getpid() async def register_process() -> None: @@ -155,11 +161,20 @@ async def register_process() -> None: startup loads the pending gate or the committed replacement before admission. A second call (boot enrolls before its migrations) keeps the state. """ - global _registered + global _registered, _process_pid, _OWN, _HEARTBEAT + if _process_pid != os.getpid(): + _process_pid = os.getpid() + _registered, _OWN, _HEARTBEAT = False, _Own(), _Heartbeat() + from server.core import cell + + cell.discard_inherited_view() + if retirement.get_gate().closed: + raise retirement.ProcessStopping("A retired participant cannot re-enroll") if not _registered: _OWN.stamp(UUID(int=0), "running") await asyncio.to_thread(_HEARTBEAT.start) _registered = True + retirement.get_gate().enrolled = True async def enroll_before_migrations(ch: Any) -> None: @@ -216,18 +231,24 @@ async def distributed_drain(transition: UUID) -> bool: async def retire_process() -> bool: - """Retire only when no admitted or registered deferred writer remains, and no - write of this process can still land on the server.""" - global _registered - if get_model_work_gate().active or not await drain_task_sets(timeout_s=0): + """Permanently close admission; certify only actually completed role work.""" + gate = retirement.get_gate() + gate.close() + _HEARTBEAT.signal_stop() + if not gate.quiescent or get_model_work_gate().active: + return False + if not await drain_task_sets(timeout_s=0): + return False + if not await flush_registered_buffers() or not await drain_task_sets(timeout_s=0): return False if not await _server_side_settled(): return False - _HEARTBEAT.signal_stop() + if _HEARTBEAT.running or await running_queries(str(process_identity())) != 0: + return False _, _, version = _OWN.stamp(UUID(int=0), "retired") + proof = retirement.guarded_proof(process_identity(), version=version) with query_tag(_TAG["log_comment"]): - await record_participant(process_identity(), UUID(int=0), "retired", version=version) - _registered = False + await record_retirement(proof) return True @@ -256,7 +277,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " "or an insert of this process still running; retire it with " - "`loom participant retire --participant %s --yes` once it stopped", + "`loom participant retire --participant %s` on its trusted host once it stopped", rmt_carry.unsettled(), process_identity(), ) @@ -268,7 +289,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: - """A boot that fails after enrolling retires its row instead of leaving one per attempt.""" + """A failed boot stops known producers but never certifies an incomplete shutdown.""" @asynccontextmanager async def wrapped(app: T) -> AsyncIterator[None]: @@ -279,6 +300,10 @@ async def wrapped(app: T) -> AsyncIterator[None]: yield except BaseException: if not started and _registered: + gate = retirement.get_gate() + gate.close() + for producer in tuple(gate.producers): + producer.cancel() try: await retire_process() except Exception as e: # noqa: BLE001 — the row expires by heartbeat instead @@ -299,11 +324,16 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: return "absent" if str(participant) == str(process_identity()): return "live" - if not row.get("expired"): + if not proves_stopped(str(row.get("host_proof") or "")): return "live" if await running_queries(participant, ch=ch) != 0: return "live" - await record_participant(participant, UUID(int=0), "retired", after=int(row["version"]), ch=ch) + proof = retirement.death_proof( + participant, str(row.get("host_proof") or ""), after=int(row["version"]) + ) + if proof is None: + return "live" + await record_retirement(proof, ch=ch) return "retired" @@ -328,9 +358,12 @@ async def retire_dead_here(*, now: float | None = None) -> int: continue if await running_queries(participant) != 0: continue - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) + if proof is None: + continue + await record_retirement(proof) log.info("provider participant %s retired: its process on this host is gone", participant) retired += 1 return retired @@ -348,10 +381,7 @@ def stale_participants() -> tuple[dict[str, str], ...] | None: async def sweep_stopped(*, now: float | None = None) -> None: - """Retire the expired participants with no query left whose Kubernetes pod is - gone (`pod_gone`), and remember the rest that block a reconfiguration (only - their process, Kubernetes or an operator can say they stopped). The workers - leader's job.""" + """Remember stale blockers; only exact verified death authorizes retirement.""" global _STALE, _last_sweep, _last_prune t = monotonic() if now is None else now if _STALE is not None and t - _last_sweep < SWEEP_S: @@ -369,11 +399,15 @@ async def sweep_stopped(*, now: float | None = None) -> None: entry = {"participant_id": participant, "heartbeat_at": str(row.get("heartbeat_at", ""))} if row.get("expired"): running = await running_queries(participant) != 0 - if not running and await pod_gone(str(row.get("host_proof") or "")): - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = ( + None + if running + else retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) - log.info("provider participant %s retired: Kubernetes deleted its pod", participant) + ) + if proof is not None: + await record_retirement(proof) continue stale.append({**entry, "reason": "queries_running" if running else "heartbeat_expired"}) _STALE = tuple(stale) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 46ec2bb9a5..b89c418e17 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -6,7 +6,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from server.core import settings_overrides +from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import set_query_tag from server.core.model_work_gate import get_model_work_gate @@ -16,6 +16,8 @@ log = logging.getLogger(__name__) +COMMAND_DRAIN_S = 15.0 # bounded cleanup; timeout leaves the participant unretired + async def _adopt_overrides(*, failing: bool = False) -> bool: """Whether the stored overrides loaded. A failure is logged when a run of them @@ -41,22 +43,44 @@ async def _keep_overrides( @asynccontextmanager -async def model_command() -> AsyncIterator[None]: - """A running command blocks a rebuild; a newborn loads the gate first. The - deployment's live-setting overrides apply before the work starts, as they do - in every server process, so a native command and the HTTP op agree.""" +async def participant_command() -> AsyncIterator[None]: + """Register all native org/principal work, including commands without models.""" + retirement.get_gate().install("command") await register_process() - keeper: asyncio.Task[None] | None = None try: - await load_active_configuration() - loaded = await _adopt_overrides() - keeper = asyncio.create_task(_keep_overrides(loaded), name="loom-cli-settings-overrides") - async with get_model_work_gate().admit(): + async with retirement.get_gate().admit(): yield finally: - if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper - if await drain_task_sets(timeout_s=15) and await flush_registered_buffers(): + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + embedder_calibration.stop_refresh_loop() + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + if producers: + await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + gate.stopped() + if await drain_task_sets(timeout_s=COMMAND_DRAIN_S) and await flush_registered_buffers(): await retire_process() + + +@asynccontextmanager +async def model_command() -> AsyncIterator[None]: + """The permanent lifecycle also holds the reversible model-work admission.""" + async with participant_command(): + keeper: asyncio.Task[None] | None = None + try: + await load_active_configuration() + loaded = await _adopt_overrides() + keeper = asyncio.create_task( + _keep_overrides(loaded), name="loom-cli-settings-overrides" + ) + async with get_model_work_gate().admit(): + yield + finally: + if keeper is not None: + keeper.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keeper diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 256a40babb..da04df218d 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -6,12 +6,18 @@ from server.core.ch import sequential_reads from server.core.ch_errors import error_code -from server.core.provider_process import host_proof, participant_tag, process_identity +from server.core.provider_process import ( + has_certificate, + participant_proof, + participant_tag, + process_identity, +) from server.core.repo_base import _CHClient, _default_client +from server.core.retirement import RetirementProof, get_gate from server.settings import settings TABLE = "loom._provider_participants" -# A participant whose last heartbeat is older than this may count as stopped. +# Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 # `expired` is decided on the server's clock, like the leases. @@ -19,7 +25,7 @@ "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL WHERE state != 'retired'" + f"FROM {TABLE} FINAL" ) ONE_SQL = ( "SELECT state, version, " @@ -58,7 +64,12 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id FROM {TABLE} FINAL WHERE state = 'retired' " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL WHERE state = 'retired' " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$') " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" @@ -66,7 +77,12 @@ def running_sql(*, inserts: bool = False) -> str: async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: with sequential_reads(): - return await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + return [ + r + for r in rows + if r["state"] != "retired" or not has_certificate(str(r.get("host_proof") or "")) + ] async def record_participant( @@ -80,9 +96,13 @@ async def record_participant( ) -> None: """`heartbeat_at` takes the server's clock (its column default); this process's own row carries its `host_proof`.""" - if state not in ("running", "drained", "retired"): + if state not in ("running", "drained"): raise ValueError("Unknown participant state") own = str(identity) == str(process_identity()) + if not own: + raise ValueError("A process writes only its own running/drained registration") + if get_gate().closed: + raise ValueError("A permanently closed participant cannot re-enroll") await (ch or _default_client()).insert( TABLE, [ @@ -94,21 +114,58 @@ async def record_participant( # A retired row is stopped at once, so every build's `provider-lock clear` # frees its holder without waiting out the heartbeat window. "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": host_proof() if own else "", + "host_proof": participant_proof() if own else "", + } + ], + ) + + +async def record_retirement(proof: RetirementProof, *, ch: _CHClient | None = None) -> None: + """The sole retired-row writer, passed evidence from the guarded lifecycle.""" + if not isinstance(proof, RetirementProof) or not has_certificate(proof.host_proof): + raise ValueError("Retirement requires verified process evidence") + await (ch or _default_client()).insert( + TABLE, + [ + { + "participant_id": proof.participant, + "transition_id": UUID(int=0), + "state": "retired", + "version": max(time_ns(), proof.after + 1, proof.version or 0), + "stale_after_s": 0, + "host_proof": proof.host_proof, } ], ) async def prune_retired(*, ch: _CHClient | None = None) -> int: - """Delete every row of the participants retired `PRUNE_AFTER_H` ago (each - process start leaves one); a process that was only paused enrolls again - with its next beat. Returns how many.""" + """Age selects already certified permanent retirements, never unknown rows. + + Retired UUIDs cannot re-enroll. An earlier in-flight beat is below the final + version; removing every version cannot resurrect that beat. + """ client = ch or _default_client() - rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) - ids = [r["participant_id"] for r in rows] - if ids: - await client.execute(_PRUNE_SQL, {"ids": ids}) + with sequential_reads(): + rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) + ids = [r["participant_id"] for r in rows if has_certificate(str(r.get("host_proof") or ""))] + if not ids: + return 0 + latest = await client.query_all( + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + expected = {str(r["participant_id"]): int(r["version"]) for r in rows} + ids = [ + r["participant_id"] + for r in latest + if r["state"] == "retired" + and int(r["version"]) == expected.get(str(r["participant_id"])) + and has_certificate(str(r.get("host_proof") or "")) + ] + if ids: + await client.execute(_PRUNE_SQL, {"ids": ids}) return len(ids) diff --git a/server/main.py b/server/main.py index 647e18b1e5..d7a3ffc2f2 100644 --- a/server/main.py +++ b/server/main.py @@ -29,6 +29,7 @@ ) from server.code.app.index.repo_modes import repo_umask from server.core import readiness as _readiness +from server.core import retirement from server.core.ch import get_client, probe_reader_user from server.core.errors import classified from server.core.logging_config import configure_logging @@ -46,6 +47,22 @@ wire() +def verify_retirement_topology() -> None: + """Capability requires both sides of the request-tail and child-task boundaries.""" + from server.core.request_tail import RequestTailMiddleware + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + classes: list[object] = [middleware.cls for middleware in app.user_middleware] + if not ( + bool(classes) + and all(cls in classes for cls in (RequestTailMiddleware, ModelMaintenanceMiddleware)) + and classes[0] is RetirementMiddleware + and classes.index(RequestTailMiddleware) < classes.index(ModelMaintenanceMiddleware) + ): + raise RuntimeError("Retirement middleware must wrap the request tail and application") + + async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None: """The bounded part of a shutdown: the instance row's stop write and the drains share one budget, and the final cleanup gets what they left unused, so the whole @@ -56,24 +73,35 @@ async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None started = time.monotonic() await _instances.stop(ch) left = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) - try: - await asyncio.wait_for(_shutdown_drains(lease, supervisor, emitter), timeout=left) - except TimeoutError: + from server.core.background import create_deferred_task + + drains = create_deferred_task( + _shutdown_drains(lease, supervisor, emitter), name="shutdown-drains" + ) + _, pending = await asyncio.wait({drains}, timeout=left) + if pending: + drains.cancel() + logging.getLogger(__name__).warning( - "shutdown drains exceeded %.0f s; retiring anyway", boot.SHUTDOWN_DRAIN_BUDGET_S + "shutdown drains exceeded %.0f s; permanent retirement remains unproved", + boot.SHUTDOWN_DRAIN_BUDGET_S, ) + else: + await drains unused = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S + unused) async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: """The lifespan's shutdown drains, in order; it bounds them all together.""" - from server.workers import leader as _leader - # An agent claim's last cutoff, which ends what an older replica's sign-in # with the code issued while the claim was landing: started first, so it # runs beside the drains below. - claims, started = asyncio.create_task(_agent_claim.drain()), time.monotonic() + from server.core.background import create_deferred_task + from server.workers import leader as _leader + + claims = create_deferred_task(_agent_claim.drain(), name="shutdown-agent-claims") + started = time.monotonic() try: await _leader.shutdown(lease, supervisor) # The service-side writes still in flight land while what they use is @@ -101,10 +129,29 @@ async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: finally: claims.cancel() # a no-op once it is done await emitter.stop() + producers = tuple(retirement.get_gate().producers) + if producers: + await asyncio.gather(*producers, return_exceptions=True) _otel.shutdown() from server.clients import _http as _http_clients await _http_clients.aclose() + retirement.get_gate().stopped() + + +async def _stop_background(probe_listener: Any) -> None: + """Stop admission and signal every producer; retirement still waits for completion.""" + from server.code.app.index import mirrors + + retirement.get_gate().close() + _readiness.begin_drain() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() + if probe_listener is not None: + await probe_listener.stop() + mirrors.stop_warmup() + embedder_calibration.stop_refresh_loop() + boot.cancel_boot_tasks() @asynccontextmanager @@ -141,6 +188,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: weak_keys=weak_keys(), ) + verify_retirement_topology() + retirement.get_gate().install("server") + track = retirement.get_gate().track from server.core.object_store import require_client require_client() @@ -174,7 +224,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_storage_authority(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - cell_loops = await boot.start_cell_gate() + from server.insights.app.provider_barrier import register_process + + await register_process() + for task in await boot.start_cell_gate(): + track(task) await boot.warn_newer_schema() await boot.warn_collection_named_namespaces() # Nothing here writes a column a migration just added while another process @@ -187,21 +241,21 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from server.core import instances as _instances from server.workers import leader as _leader - instances_loop = await _instances.start( - ch, - _leader.leads_deployment, - label=_instance_mod.instance_label() or "", - workers_lease=_leader.advertised_lease, - on_beat=_leader.read_holder, + track( + await _instances.start( + ch, + _leader.leads_deployment, + label=_instance_mod.instance_label() or "", + workers_lease=_leader.advertised_lease, + on_beat=_leader.read_holder, + ) ) from server.core.background import register_buffer_drainer - from server.insights.app.provider_barrier import register_process register_buffer_drainer("boot-writes", boot.drain_writes) from server.insights.app.provider_runtime import load_active_configuration, refresh_loop - await register_process() await load_active_configuration() emitter = get_emitter() @@ -234,24 +288,29 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: boot.publish_deployment_config(ch) # This process verifies tokens: it loads the revoked set and the signing # keyring now and keeps both fresh. - auth_loops = await boot.start_token_state() + for task in await boot.start_token_state(): + track(task) probe_listener = boot.start_probe_listener() - overrides_loop = boot.start_runtime_overrides() + track(boot.start_runtime_overrides()) await probe_reader_user(ch) from server.insights.app import analysis as _analysis # Provisioned and probed in the background: the lease can wait for minutes. - analysis_loop = _analysis.start_probe_loop() + track(_analysis.start_probe_loop()) from server.assistant.app import service as _assistant assistant_loop = _assistant.start_setup_loop() - tracker_sources_loop = boot.start_tracker_sources_refresher(ch) + if assistant_loop is not None: + track(assistant_loop) + track(boot.start_tracker_sources_refresher(ch)) from server.code.app.index import repo_registry as _repo_registry - code_repos_loop = await _repo_registry.start_refresher(ch) + track(await _repo_registry.start_refresher(ch)) from server.code.app.index import mirrors as _mirrors - _mirrors.start_warmup() # this host's git mirrors, in the background + mirror_loop = _mirrors.start_warmup() + if mirror_loop is not None: + retirement.get_gate().track(mirror_loop) boot.publish_frontend_assets(ch) boot.mark_process_start(ch) await boot.probe_sandbox() @@ -269,37 +328,22 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: "WorkerSupervisor: not started in pid %s — %s", os.getpid(), lease.reason ) lease_loop = boot.start_lease_loop(lease, supervisor, ch) + if lease_loop is not None: + track(lease_loop) if not get_model_work_gate().closed: await boot.provider_report() await boot.probe_vector_capabilities(ch) await boot.calibrate_similarity_gates() - provider_refresh = asyncio.create_task(refresh_loop(), name="provider-configuration-refresh") + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")) # SIGTERM: not ready first, the port closes LOOM_SHUTDOWN_READY_DELAY_S later. boot.install_drain_on_sigterm() try: yield finally: - _readiness.begin_drain() - provider_refresh.cancel() - await asyncio.gather(provider_refresh, return_exceptions=True) - if lease_loop is not None: - lease_loop.cancel() - for loop in (*cell_loops, *auth_loops, overrides_loop): - loop.cancel() - if probe_listener is not None: - await probe_listener.stop() - tracker_sources_loop.cancel() - code_repos_loop.cancel() - analysis_loop.cancel() - if assistant_loop is not None: - assistant_loop.cancel() - _mirrors.stop_warmup() - embedder_calibration.stop_refresh_loop() - instances_loop.cancel() - boot.cancel_boot_tasks() + await _stop_background(probe_listener) await _shut_down(ch, lease, supervisor, emitter) await boot.close_storage_authority() @@ -495,6 +539,10 @@ def attach_browser_middleware(target: FastAPI) -> None: http_transport.attach(app) mcp_transport.attach(app) +from server.transport.retirement import RetirementMiddleware # noqa: E402 + +app.add_middleware(RetirementMiddleware) + from server.auth import login as _login # noqa: E402 from server.auth import oauth as _oauth # noqa: E402 from server.auth import scim_routes as _scim_routes # noqa: E402 diff --git a/server/memory/app/embedder_calibration.py b/server/memory/app/embedder_calibration.py index cc9569ba6b..8ddf1f4eb5 100644 --- a/server/memory/app/embedder_calibration.py +++ b/server/memory/app/embedder_calibration.py @@ -592,7 +592,11 @@ async def _loop() -> None: if not get_model_work_gate().closed: await refresh_shared() - _refresh_task = asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + from server.core.retirement import get_gate + + _refresh_task = get_gate().track( + asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + ) return _refresh_task diff --git a/server/observability/events.py b/server/observability/events.py index da4de2c333..a89e648a0c 100644 --- a/server/observability/events.py +++ b/server/observability/events.py @@ -194,8 +194,9 @@ async def start(self) -> None: # Process-lifetime task: it must not inherit the tag, request id or # auth of whichever request first started it. from server.core.background import detached_task + from server.core.retirement import get_gate - self._task = detached_task(self._drain_loop(), name="loom-events-drain") + self._task = get_gate().track(detached_task(self._drain_loop(), name="loom-events-drain")) async def stop(self) -> None: self._running = False diff --git a/server/observability/problems.py b/server/observability/problems.py index 07571de3c6..6e43eccfc8 100644 --- a/server/observability/problems.py +++ b/server/observability/problems.py @@ -936,8 +936,9 @@ def _participant_problems(state: HealthState) -> list[Problem]: f"A provider reconfiguration would wait on these participants: {ids}. " "`heartbeat_expired` ones stopped heartbeating (they may be paused), " "and `queries_running` ones did so with a query still running.", - "Once a participant's process is confirmed stopped, retire it with " - "`loom participant retire --participant --yes`.", + "On its trusted host, verify exact physical death and settled queries with " + "`loom participant retire --participant `. Expiry or a deleted pod " + "object does not prove death.", source="runtime", ) ] diff --git a/server/transport/model_maintenance.py b/server/transport/model_maintenance.py index 7f59ba2e2c..f27092672d 100644 --- a/server/transport/model_maintenance.py +++ b/server/transport/model_maintenance.py @@ -6,7 +6,7 @@ from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send -from server.core import cell, deployment_config +from server.core import cell, deployment_config, retirement from server.core.errors import classified from server.core.model_work_gate import ModelChangeInProgress, ModelWorkGate, get_model_work_gate from server.settings import settings @@ -111,6 +111,20 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] not in ("http", "websocket"): await self.app(scope, receive, send) return + try: + async with retirement.get_gate().admit(): + await self._dispatch(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) + + async def _dispatch(self, scope: Scope, receive: Receive, send: Send) -> None: if (reason := cell.gate_reason()) and not _health_probe(scope): await _refuse_cell(scope, receive, send, reason) return diff --git a/server/transport/retirement.py b/server/transport/retirement.py new file mode 100644 index 0000000000..396a304554 --- /dev/null +++ b/server/transport/retirement.py @@ -0,0 +1,28 @@ +"""Outer admission includes request-tail settlement after inner dispatch ends.""" + +from starlette.types import ASGIApp, Receive, Scope, Send + +from server.core import retirement +from server.transport.model_maintenance import _refuse + + +class RetirementMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + try: + async with retirement.get_gate().admit(): + await self.app(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) diff --git a/server/worker_main.py b/server/worker_main.py index 6854be4082..c265c7c124 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -20,6 +20,7 @@ from typing import Any from server import boot +from server.core import retirement from server.settings import settings from server.workers import child, leader from server.workers.supervisor import get_supervisor @@ -46,20 +47,24 @@ async def _trackers() -> None: # `with`, not set_query_tag: the supervisor and the shutdown run in this context after. with query_tag("boot.workers_child"): register_buffer_drainer("boot-writes", boot.drain_writes) + retirement.get_gate().install("worker") await register_process() await load_active_configuration() await get_emitter().start() await registry_sources.refresh_db_sources(force=True, ch=ch) if not get_model_work_gate().closed: await boot.calibrate_similarity_gates() + track = retirement.get_gate().track tasks = [ - asyncio.create_task(refresh_loop(), name="provider-configuration-refresh"), - asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh"), - await repo_registry.start_refresher(ch), + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")), + track(asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh")), + track(await repo_registry.start_refresher(ch)), # The workers read LIVE settings too: their passes see the deployment's overrides. - boot.start_runtime_overrides(), + track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) + for task in tasks: + retirement.get_gate().track(task) return tasks @@ -113,13 +118,18 @@ async def _serve(fd: int, alive_fd: int) -> None: supervisor.start_all() await stop.wait() log.info("workers child %s stopping", os.getpid()) + retirement.get_gate().close() embedder_calibration.stop_refresh_loop() for task in services: task.cancel() await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() + retirement.get_gate().stopped() finally: + retirement.get_gate().close() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() beat.cancel() await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S) await boot.close_storage_authority() diff --git a/server/workers/supervisor.py b/server/workers/supervisor.py index f951132452..b2b08946c5 100644 --- a/server/workers/supervisor.py +++ b/server/workers/supervisor.py @@ -30,6 +30,7 @@ from typing import Any from server.core.model_work_gate import get_model_work_gate +from server.core.retirement import ProcessStopping, get_gate from server.observability.traffic_class import bind_background from server.settings import settings from server.workers import leader, pass_context, status_store @@ -139,6 +140,7 @@ def _spawn(self, spec: WorkerSpec) -> None: _publish_streak(spec.name, 0) _publish_partial(spec.name, 0) task = asyncio.create_task(self._supervise_as_background(spec), name=_task_name(spec)) + get_gate().track(task) self._tasks.append(task) def _restart(self, spec: WorkerSpec, _ended: asyncio.Task[None]) -> None: @@ -257,7 +259,7 @@ async def _supervise(self, spec: WorkerSpec) -> None: try: # A pass past its deadline is cancelled and counted as a # failure (TimeoutError). - async with get_model_work_gate().admit(wait=True): + async with get_gate().admit(), get_model_work_gate().admit(wait=True): # Maintenance waiting consumes neither the pass deadline # nor its duration/accounting. An admitted pass finishes # normally; changing models never cancels it mid-write. @@ -278,6 +280,8 @@ async def _supervise(self, spec: WorkerSpec) -> None: spec, ctx, partial=bool(ctx.count), streak=consecutive_failures ) ) + except ProcessStopping: + return except pass_context.PassStopped: # The pass saw a stop or a stale lease between batches. Recorded # like a cancel; the loop top waits until the lease is fresh. diff --git a/tests/fixtures/provider_process_before_retirement.py b/tests/fixtures/provider_process_before_retirement.py new file mode 100644 index 0000000000..6ef0d6af5a --- /dev/null +++ b/tests/fixtures/provider_process_before_retirement.py @@ -0,0 +1,173 @@ +"""This process's provider-participant id and the tag its ClickHouse queries carry. + +Every ClickHouse client the process opens names the id in its HTTP User-Agent, +so `system.processes` says whether a participant that stopped heartbeating +still has a query running. +""" + +import asyncio +import os +import re +import socket +from pathlib import Path +from uuid import UUID, uuid4 + +_identity: tuple[int, UUID] | None = None + + +def process_identity() -> UUID: + """One id per process; a forked child draws its own.""" + global _identity + if _identity is None or _identity[0] != os.getpid(): + _identity = (os.getpid(), uuid4()) + return _identity[1] + + +def participant_tag(identity: UUID | str) -> str: + """What `system.processes.http_user_agent` contains for that participant's queries.""" + return f"loom-participant/{UUID(str(identity)).hex}" + + +def client_name() -> str: + """The `client_name` every ClickHouse client of this process is opened with.""" + return participant_tag(process_identity()) + + +# host name | kernel boot id | container | pid namespace | pid | process start +# (clock ticks after boot) | Kubernetes namespace and pod name (POD_NAMESPACE, +# POD_NAME: the host name is the pod name cut to 63 characters; '' elsewhere) +_proof: tuple[int, str] | None = None +# Docker mounts /etc/hostname from its container's directory; a Kubernetes container +# mounts its termination log from the pod's, under the container's name. +_CONTAINER = ( + re.compile(r"/containers/([0-9a-f]{64})/hostname "), + re.compile(r"/pods/([0-9a-f-]{36})/containers/([^/ ]+)/"), +) + + +def _start_ticks(pid: int | str) -> str: + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + return f.read().rsplit(")", 1)[1].split()[19] + + +def _boot_and_pidns() -> tuple[str, str]: + with open("/proc/sys/kernel/random/boot_id", encoding="ascii") as f: + boot = f.read().strip() + return boot, str(os.stat("/proc/self/ns/pid").st_ino) + + +def _container() -> str: + """This container's id, stable across its restarts; '' outside one, or where + the mounts do not say.""" + try: + with open("/proc/self/mountinfo", encoding="utf-8") as f: + mounts = f.read() + except OSError: + return "" + for pattern in _CONTAINER: + found = pattern.search(mounts) + if found: + return "/".join(found.groups()) + return "" + + +def host_proof() -> str: + """What lets a later process on this host prove this one stopped (`proves_stopped`); + '' where /proc does not say.""" + global _proof + if _proof is None or _proof[0] != os.getpid(): + try: + boot, pidns = _boot_and_pidns() + value = "|".join( + ( + socket.gethostname(), + boot, + _container(), + pidns, + str(os.getpid()), + _start_ticks("self"), + os.environ.get("POD_NAMESPACE", ""), + os.environ.get("POD_NAME", ""), + ) + ) + except (OSError, IndexError): + value = "" + _proof = (os.getpid(), value) + return _proof[1] + + +def proves_stopped(proof: str) -> bool: + """Whether the process `proof` names has stopped, judged from this one: same host + name and kernel boot, and either its pid is free or runs another process than + the one that started then (same pid namespace), or it ran in this container + before the container restarted into a new pid namespace (which kills every + process of the old one). Two containers that share a host name never prove + each other stopped; a process that cannot be inspected is not proven stopped.""" + parts = proof.split("|") + if len(parts) not in (6, 7, 8): + return False + host, boot, container, pidns, pid, start = parts[:6] + try: + my_boot, my_pidns = _boot_and_pidns() + except OSError: + return False + if host != socket.gethostname() or boot != my_boot: + return False + if pidns != my_pidns: + return bool(container) and container == _container() + # Only positive evidence: no such process, or another one under its pid. + # A pid this process may not inspect (hidepid, a permission error) proves nothing. + try: + os.kill(int(pid), 0) + except ProcessLookupError: + return True + except (OSError, ValueError): + return False + try: + return _start_ticks(pid) != start + except (OSError, IndexError): + return False + + +_SA = "/var/run/secrets/kubernetes.io/serviceaccount" +_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") + + +async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: + """Whether Kubernetes says the pod `proof` names is gone: no pod of that name + in its namespace, one with another UID, or this one in phase Failed or + Succeeded (an evicted pod keeps its object; a pod that restarts its + containers reaches those phases only once every one stopped for good). The + kubelet removes a pod's object once its containers stopped (or an operator + force-deleted it), so each is positive evidence a lost pod's process ended. + Asked only of this pod's own namespace, with its service-account token; + False wherever the API or the proof does not say.""" + parts = proof.split("|") + found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None + namespace = os.environ.get("POD_NAMESPACE", "") + if not found or not namespace or parts[6] != namespace or not parts[7]: + return False + host = os.environ.get("KUBERNETES_SERVICE_HOST", "") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + try: + token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() + except OSError: + return False + if not host or not token: + return False + import httpx + + url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" + try: + async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: + r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + except (httpx.HTTPError, OSError): + return False + if r.status_code == 404: + return True + if r.status_code != 200: + return False + pod = r.json() + if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): + return True + return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py new file mode 100644 index 0000000000..0652b4403f --- /dev/null +++ b/tests/integration/test_cell_retirement.py @@ -0,0 +1,103 @@ +"""Certified retirement and pruning against an owned scratch ClickHouse table.""" + +from __future__ import annotations + +import asyncio +import os +import sys +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process, retirement +from server.core.ch import CHClient, query_tag +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( + monkeypatch, +): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + database = "retirement_" + uuid4().hex + table = database + ".participants" + client = CHClient() + await client.connect() + with query_tag("test.cell_retirement"): + await client.execute("CREATE DATABASE {database:Identifier}", {"database": database}) + try: + await client.execute( + "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", + {"database": database}, + ) + old_table = store.TABLE + for name in ( + "TABLE", + "PARTICIPANTS_SQL", + "OWN_VERSION_SQL", + "_PRUNE_IDS_SQL", + "_PRUNE_SQL", + ): + monkeypatch.setattr(store, name, getattr(store, name).replace(old_table, table)) + monkeypatch.setattr( + store, "_PRUNE_SQL", store._PRUNE_SQL + " SETTINGS mutations_sync=2" + ) + stopped, legacy = uuid4(), uuid4() + process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") + await asyncio.wait_for(process.wait(), timeout=10) + fields = provider_process.host_proof().split("|") + fields[4:6] = [str(process.pid), "1"] + identity = "|".join(fields) + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 1, + "host_proof": identity, + }, + {"participant_id": legacy, "state": "retired", "version": 1, "host_proof": ""}, + ], + ) + assert await store.running_queries(stopped, ch=client) == 0 + proof = retirement.death_proof(stopped, identity, after=1) + assert proof is not None + await store.record_retirement(proof, ch=client) + # A previously in-flight heartbeat cannot outrank the certified final row. + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 2, + "host_proof": identity, + } + ], + ) + rows = await client.query_all( + f"SELECT participant_id, transition_id, state, version, stale_after_s, host_proof FROM {table} FINAL" + ) + final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) + assert str(final["transition_id"]) == str(UUID(int=0)) + assert final["state"] == "retired" and final["version"] > 2 + assert final["stale_after_s"] == 0 + assert final["host_proof"] == provider_process.certified(identity) + assert [str(row["participant_id"]) for row in await store.participants(ch=client)] == [ + str(legacy) + ] + # Both are old, but only the certified identity is allowed to disappear. + await client.execute( + f"ALTER TABLE {table} UPDATE heartbeat_at=now64(3)-toIntervalHour(25) " + "WHERE 1 SETTINGS mutations_sync=2" + ) + assert await store.prune_retired(ch=client) == 1 + rows = await client.query_all(f"SELECT participant_id, state FROM {table} FINAL") + assert [(str(row["participant_id"]), row["state"]) for row in rows] == [ + (str(legacy), "retired") + ] + finally: + await client.execute("DROP DATABASE {database:Identifier} SYNC", {"database": database}) + await client.disconnect() diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 9a6c9db1e2..16c9704fa8 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -19,11 +19,14 @@ def _live() -> None: @pytest.mark.asyncio -async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): +async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(monkeypatch): _live() ch = _default_client() await ch.connect() identity, transition = uuid4(), uuid4() + monkeypatch.setattr( + "server.insights.store.provider_participants.process_identity", lambda: identity + ) try: await record_participant(identity, transition, "running", ch=ch) await record_participant(identity, transition, "drained", ch=ch) @@ -39,10 +42,11 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): rows = [r for r in await participants(ch=ch) if str(r["participant_id"]) == str(identity)] assert rows[0]["expired"] assert await running_queries(identity, ch=ch) == 0 - await record_participant( - identity, transition, "retired", after=int(rows[0]["version"]), ch=ch - ) - assert all(str(row["participant_id"]) != str(identity) for row in await participants(ch=ch)) + with pytest.raises(ValueError, match="state"): + await record_participant( + identity, transition, "retired", after=int(rows[0]["version"]), ch=ch + ) + assert any(str(row["participant_id"]) == str(identity) for row in await participants(ch=ch)) finally: try: await ch.execute( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ad5291a171..7a1f0ded28 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -80,6 +80,7 @@ async def checked(_ch): return None monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(provider_command, "participant_command", command) monkeypatch.setattr(pg_profile, "check_configured", checked) @@ -1152,3 +1153,11 @@ async def _none(org: str, namespaces: list[str], **_: object) -> dict[str, str]: return {} monkeypatch.setattr("server.knowledge.app.mirrors.mirrors_of", _none) + + +@pytest.fixture(autouse=True) +def isolated_retirement_gate(monkeypatch): + """Each test models a fresh process; permanent shutdown cannot leak between them.""" + from server.core import retirement + + monkeypatch.setattr(retirement, "_gate", retirement.RetirementGate()) diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..37e0e5e39e 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -185,12 +185,12 @@ ], "loom participant": [ "usage: loom participant [-h] {retire} ...", - "positional arguments: {retire} retire Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "positional arguments: {retire} retire Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "options: -h, --help show this help message and exit" ], "loom participant retire": [ - "usage: loom participant retire [-h] --participant PARTICIPANT [--yes]", - "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire --yes Its process is confirmed stopped" + "usage: loom participant retire [-h] --participant PARTICIPANT", + "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire" ], "loom deployment": [ "usage: loom deployment [-h] {publish} ...", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..8216d4de36 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -5682,8 +5682,7 @@ "description": null, "dests": [ "help", - "participant", - "yes" + "participant" ] } ], @@ -5719,21 +5718,6 @@ "required": true, "help": "The participant id to retire", "metavar": null - }, - { - "option_strings": [ - "--yes" - ], - "dest": "yes", - "action": "_StoreTrueAction", - "default": false, - "const": true, - "choices": null, - "nargs": 0, - "type": null, - "required": false, - "help": "Its process is confirmed stopped", - "metavar": null } ] } @@ -5744,7 +5728,7 @@ "choices_help": [ { "dest": "retire", - "help": "Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "help": "Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "metavar": "retire" } ] diff --git a/tests/unit/test_analytics_export.py b/tests/unit/test_analytics_export.py index 7967d8edec..79412dc567 100644 --- a/tests/unit/test_analytics_export.py +++ b/tests/unit/test_analytics_export.py @@ -252,7 +252,10 @@ async def disconnect(self) -> None: def test_an_export_of_yesterday_connects_inside_its_deadline_and_hashes_under_the_key( - now: datetime, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + stub_model_command, + now: datetime, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: from server.core import ch as ch_module from server.insights.app import analytics_export as app diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 60abc95312..6d14350e03 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -17,7 +17,13 @@ from server.settings import settings ROOT = Path(__file__).resolve().parents[2] -_CONNECTS = {"run_connected", "connect_checked", "connect_declared", "_connected"} +_CONNECTS = { + "run_connected", + "connect_checked", + "connect_declared", + "check_acceptance", + "_connected", +} # `.connect()` receivers that are not the deployment's ClickHouse writer: the setup # command's own admin client and SQLite files an importer reads. _OTHER_CLIENTS = {"preflight", "sqlite3"} @@ -969,6 +975,7 @@ def test_a_cell_whose_rows_cannot_be_read_refuses_every_declared_org( asyncio.run(_ch.check_orgs([])) +@pytest.mark.usefixtures("stub_model_command") def test_a_command_for_an_org_the_cell_does_not_hold_never_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py new file mode 100644 index 0000000000..a5a3c8dcce --- /dev/null +++ b/tests/unit/test_cell_retirement.py @@ -0,0 +1,752 @@ +"""Permanent retirement requires completion, never expiry or an API object's absence.""" + +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_expired_quiet_participant_is_not_manual_death(monkeypatch): + participant = uuid4() + row = { + "participant_id": participant, + "transition_id": UUID(int=0), + "version": 7, + "state": "running", + "expired": True, + "host_proof": provider_process.host_proof(), + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_participant", AsyncMock()) + assert await barrier.retire_participant(participant) == "live" + barrier.record_participant.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_store_cannot_manufacture_retirement(): + ch = AsyncMock() + with pytest.raises(ValueError, match="[Rr]etire|[Ss]tate"): + await store.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) + ch.insert.assert_not_awaited() + + +def test_changed_namespace_does_not_prove_old_container_ended(monkeypatch): + fields = provider_process.host_proof().split("|") + assert len(fields) == 8 + fields[2] = "c" * 64 + fields[3] = "another-namespace" + monkeypatch.setattr(provider_process, "_container", lambda: fields[2]) + assert not provider_process.proves_stopped("|".join(fields)) + + +@pytest.mark.asyncio +async def test_close_is_permanent_through_an_awaited_retirement_write(monkeypatch): + import asyncio + + from server.core import retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + entered, finish = asyncio.Event(), asyncio.Event() + + async def write(proof): + assert gate.quiescent + entered.set() + await finish.wait() + + monkeypatch.setattr(barrier, "record_retirement", write) + monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + task = asyncio.create_task(barrier.retire_process()) + await entered.wait() + with pytest.raises(retirement.ProcessStopping): + async with gate.admit(): + pytest.fail("Permanent closure must not reopen while insert is awaited") + with pytest.raises(retirement.ProcessStopping): + await barrier.register_process() + finish.set() + assert await task + + +def _request_app(handler): + from fastapi import FastAPI + + from server.main import attach_browser_middleware + from server.transport import http + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + app = FastAPI() + app.add_api_route("/oauth/token", handler, methods=["POST"]) + app.add_middleware(ModelMaintenanceMiddleware) + attach_browser_middleware(app) + http.attach(app) + app.add_middleware(RetirementMiddleware) + return app + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_tail", [False, True]) +async def test_composed_request_tail_holds_counter_after_dispatch_raises(monkeypatch, cancel_tail): + import asyncio + + import httpx + + from server.core import request_tail, retirement + + gate = retirement.get_gate() + entered, finish, canceled = asyncio.Event(), asyncio.Event(), asyncio.Event() + observed = [] + + async def write(): + observed.append(gate.active) + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() # cancellation requested; this writer still exists + + async def handler(): + assert gate.active == 2 + assert request_tail.defer(write) + raise ValueError("before response start") + + app = _request_app(handler) + transport = httpx.ASGITransport(app=app, raise_app_exceptions=False) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client: + task = asyncio.create_task(client.post("/oauth/token")) + await asyncio.wait_for(entered.wait(), 5) + assert gate.active == 1 # actual handler ended, outer tail remains counted + assert observed == [1] + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + assert not task.done() + barrier.record_retirement.assert_not_awaited() + if cancel_tail: + task.cancel() + await asyncio.wait_for(canceled.wait(), 5) + assert gate.active == 1 and not task.done() + assert not await barrier.retire_process() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_inner_scope_refuses_when_close_lands_after_outer_admission(): + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + handler, send = AsyncMock(), AsyncMock() + + async def between(scope, receive, send): + assert gate.active == 1 + gate.close() + await ModelMaintenanceMiddleware(handler)(scope, receive, send) + assert gate.active == 1 + + await RetirementMiddleware(between)( + {"type": "http", "path": "/oauth/token", "method": "POST"}, AsyncMock(), send + ) + handler.assert_not_awaited() + assert send.await_args_list[0].args[0]["status"] == 503 + assert gate.active == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + [ + "/oauth/token", + "/oauth/callback", + "/v1/auth/login", + "/v1/scim/Users", + "/v1/admin/models/reset", + ], +) +async def test_canceled_control_dispatch_remains_counted_until_actual_completion(path): + import asyncio + + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + entered, canceled, finish = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def handler(scope, receive, send): + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + assert gate.active == 2 # work could still mint/write here + + task = asyncio.create_task( + RetirementMiddleware(ModelMaintenanceMiddleware(handler))( + {"type": "http", "path": path, "method": "POST"}, AsyncMock(), AsyncMock() + ) + ) + await entered.wait() + task.cancel() + await canceled.wait() + gate.close() + assert gate.active == 2 and not task.done() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_registered_descendant_blocks_after_outer_and_inner_completed(monkeypatch): + import asyncio + + import httpx + + from server.core import retirement + from server.core.background import create_deferred_task + + gate = retirement.get_gate() + entered, finish = asyncio.Event(), asyncio.Event() + tasks = [] + + async def descendant(): + entered.set() + await finish.wait() + + async def handler(): + tasks.append(create_deferred_task(descendant(), name="retirement-test-descendant")) + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + assert (await client.post("/oauth/token")).status_code == 200 + await entered.wait() + assert gate.active == 0 + gate.install("test") + gate.enrolled = True + gate.stopped() + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + assert not await barrier.retire_process() + barrier.record_retirement.assert_not_awaited() + finish.set() + await asyncio.gather(*tasks) + + +@pytest.mark.asyncio +async def test_certificates_fail_closed_in_original_parsers(): + from pathlib import Path + from types import ModuleType + + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + previous = ModuleType("old_provider_process") + exec(compile(source, "old_provider_process.py", "exec"), previous.__dict__) + proof = provider_process.certified(provider_process.host_proof()) + assert len(proof.split("|")) == 9 + assert not previous.proves_stopped(proof) + assert not await previous.pod_gone(proof) + assert provider_process.has_certificate(proof) + assert not provider_process.has_certificate(proof.replace("cell-retirement-v1", "unknown-v2")) + assert not provider_process.has_certificate("||||||||cell-retirement-v1") + + +@pytest.mark.asyncio +async def test_uncertified_retired_rows_remain_census_blockers_and_unprunable(): + from types import SimpleNamespace + + identity = uuid4() + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": ""} + ch = SimpleNamespace(query_all=AsyncMock(return_value=[row]), execute=AsyncMock()) + assert await store.participants(ch=ch) == [row] + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_prune_rechecks_latest_certificate_and_version(): + from types import SimpleNamespace + + identity = uuid4() + proof = provider_process.certified(provider_process.host_proof()) + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": proof} + newer = {**row, "state": "running", "version": 8} + ch = SimpleNamespace(query_all=AsyncMock(side_effect=[[row], [newer]]), execute=AsyncMock()) + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +def test_production_topology_installs_outer_and_inner_scopes(): + from server.main import verify_retirement_topology + + verify_retirement_topology() + + +@pytest.mark.parametrize("migrate", [False, True]) +@pytest.mark.asyncio +async def test_server_enrolls_after_pair_verification_before_first_cell_read(monkeypatch, migrate): + from types import SimpleNamespace + + from server import boot, main + from server.auth import jwt + from server.core import object_store + from server.observability import instance + from server.settings import settings + + events = [] + + class FirstRead(Exception): + pass + + async def paired(*args, **kwargs): + events.append("paired") + + async def enrolled(): + assert events[-1] == "paired" + events.append("enrolled") + + async def first_read(): + assert events[-1] == "enrolled" + events.append("read") + raise FirstRead + + async def migrations(*args, enabled, **kwargs): + assert enabled is migrate + return 0 # disabled and no-pending-migrations boots both need enrollment + + monkeypatch.setattr(instance, "require_instance_label", lambda *args, **kwargs: None) + monkeypatch.setattr(jwt, "require_real_signing_secret", lambda **kwargs: None) + monkeypatch.setattr(boot, "require_identity_settings", lambda: None) + monkeypatch.setattr(boot, "require_safe_bind", lambda **kwargs: None) + monkeypatch.setattr(object_store, "require_client", lambda: None) + monkeypatch.setattr(main, "get_client", lambda: SimpleNamespace(connect=AsyncMock())) + monkeypatch.setattr(boot, "require_clickhouse_ready", AsyncMock()) + monkeypatch.setattr(boot, "register_readiness_checks", lambda ch: None) + monkeypatch.setattr(boot, "run_fenced_boot_migrations", migrations) + monkeypatch.setattr(boot, "require_keeper", AsyncMock()) + monkeypatch.setattr(boot, "require_storage_authority", paired) + monkeypatch.setattr(boot, "start_cell_gate", first_read) + monkeypatch.setattr(barrier, "register_process", enrolled) + monkeypatch.setattr(settings, "boot_migrate", migrate) + with pytest.raises(FirstRead): + async with main.lifespan(main.app): + pytest.fail("Only the first read is under test") + assert events == ["paired", "enrolled", "read"] + + +def test_cli_connects_verified_then_enrolls_then_reads_acceptance(monkeypatch): + from contextlib import asynccontextmanager + from types import SimpleNamespace + + from server.cli import _ch + from server.core import pg_profile + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verified(ch): + assert ch is client + events.append("verified") + + @asynccontextmanager + async def command(): + events.append("enrolled") + yield + events.append("completed") + + async def accepting(orgs): + assert events == ["verified", "enrolled"] + events.append("read") + + async def work(ch): + assert events[-1] == "read" + events.append("work") + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(_ch, "check_orgs", accepting) + assert _ch.run_connected(work, orgs=("example",)) == 7 + assert events == ["verified", "enrolled", "read", "work", "completed"] + + +def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census(): + import ast + from pathlib import Path + + root = Path(__file__).parents[2] / "server" + constructors, writers, commands, stopped, plain = set(), set(), set(), set(), set() + for path in root.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else "" + ) + relative = str(path.relative_to(root)) + if name == "RetirementProof": + constructors.add(relative) + if name == "record_retirement": + writers.add(relative) + if name == "model_command": + commands.add(relative) + if name == "participant_command": + plain.add(relative) + if name == "stopped": + stopped.add(relative) + assert constructors == {"core/retirement.py"} + assert writers == {"insights/app/provider_barrier.py"} + assert commands == { + "cli/_ch.py", + "cli/code_run.py", + "cli/import_clickgap.py", + "cli/import_recall.py", + } + assert plain == { + "cli/sql_access.py", + "cli/analytics_export.py", + "cli/maintenance.py", + "insights/app/provider_command.py", + } + assert stopped == {"main.py", "worker_main.py", "insights/app/provider_command.py"} + + +def test_manual_retirement_has_no_affirmation_bypass(): + import argparse + + from server.cli.participant import register_participant_cli + + parser = argparse.ArgumentParser() + register_participant_cli(parser.add_subparsers(dest="command", required=True)) + with pytest.raises(SystemExit) as error: + parser.parse_args(["participant", "retire", "--participant", str(uuid4()), "--yes"]) + assert error.value.code == 2 + + +def test_fork_cannot_inherit_admission(monkeypatch): + from server.core import retirement + + gate = retirement.get_gate() + gate.install("server") + gate.enrolled = True + monkeypatch.setattr(retirement, "_pid", -1) + child = retirement.get_gate() + assert child is not gate and child.role == "server" and not child.enrolled + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_state", ["404", "replacement", "Failed", "Succeeded", "unreachable"]) +async def test_pod_object_death_never_retires_work_that_can_resume( + monkeypatch, tmp_path, api_state +): + import asyncio + from pathlib import Path + from types import ModuleType + + import httpx + + from server.core import retirement + + old = ModuleType("old_reaper") + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + exec(compile(source, "old_reaper.py", "exec"), old.__dict__) + uid = str(uuid4()) + proof = "|".join( + ("partitioned-host", "boot", uid + "/loom", "4026", "7", "99", "loom", "old-pod") + ) + (tmp_path / "token").write_text("test-token") + monkeypatch.setattr(old, "_SA", str(tmp_path)) + monkeypatch.setenv("POD_NAMESPACE", "loom") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "kubernetes.invalid") + seen = [] + + def answer(request): + seen.append(request) + if api_state == "unreachable": + raise httpx.ConnectError("partitioned", request=request) + return httpx.Response( + 404 if api_state == "404" else 200, + json={ + "metadata": {"uid": str(uuid4()) if api_state == "replacement" else uid}, + "status": {"phase": api_state}, + }, + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real_client(transport=httpx.MockTransport(answer)) + ) + # Demonstrate the exact old mistake before testing its replacement. + assert await old.pod_gone(proof) is (api_state != "unreachable") + participant = uuid4() + row = { + "participant_id": participant, + "state": "running", + "version": 9, + "expired": True, + "host_proof": proof, + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + monkeypatch.setattr(barrier, "_STALE", None) + resume, admitted = asyncio.Event(), asyncio.Event() + effects = [] + + async def paused(): + async with retirement.get_gate().admit(): + admitted.set() + await resume.wait() + effects.append("mint after object deletion") + + task = asyncio.create_task(paused()) + await admitted.wait() + await barrier.sweep_stopped(now=5000) + barrier.record_retirement.assert_not_awaited() + assert barrier.stale_participants()[0]["participant_id"] == str(participant) + assert len(seen) == 1 # the current reaper never asks the API for a death certificate + resume.set() + await task + assert effects == ["mint after object deletion"] + + +@pytest.mark.asyncio +async def test_production_child_task_cancellation_stays_counted_until_completion(): + import asyncio + import contextlib + + import anyio + import httpx + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + entered.set() + with anyio.CancelScope(shield=True): + await finish.wait() + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await entered.wait() + task.cancel() + # Let BaseHTTPMiddleware propagate cancellation to its downstream child. + for _ in range(5): + await asyncio.sleep(0) + gate.close() + assert gate.active > 0 + assert not task.done() + finish.set() + with contextlib.suppress(asyncio.CancelledError): + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_shutdown_budget_does_not_wait_for_cancellation_deferring_drains(monkeypatch): + import asyncio + import time + + from server import boot, main + from server.core import instances, retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + canceled, finish = asyncio.Event(), asyncio.Event() + owned = [] + + async def drains(*args): + owned.append(asyncio.current_task()) + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + + async def leave(*args, **kwargs): + assert not await barrier.retire_process() + + monkeypatch.setattr(instances, "stop", AsyncMock()) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 0.01) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + started = time.monotonic() + await asyncio.wait_for(main._shut_down(None, None, None, None), timeout=0.5) + assert time.monotonic() - started < 0.5 + await canceled.wait() + assert not owned[0].done() + barrier.record_retirement.assert_not_awaited() + finish.set() + await owned[0] + + +def test_nonmodel_principal_command_enrolls_before_acceptance(monkeypatch): + from types import SimpleNamespace + + from server.cli import _ch, sql_access + from server.core import pg_profile, retirement + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verify(ch): + events.append("verified") + + async def register(): + assert events == ["verified"] + retirement.get_gate().enrolled = True + events.append("registered") + + async def acceptance(orgs): + assert events == ["verified", "registered"] + events.append("acceptance") + + async def work(ch): + assert events[-1] == "acceptance" + assert retirement.get_gate().active == 1 + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(provider_command, "register_process", register) + monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + assert sql_access._connected(work, orgs=("example",)) == 7 + assert retirement.get_gate().closed and retirement.get_gate().active == 0 + + +@pytest.mark.asyncio +async def test_stream_and_response_background_cleanup_remain_counted(monkeypatch): + import asyncio + + import httpx + from starlette.background import BackgroundTask + from starlette.responses import StreamingResponse + + from server.core import retirement + + gate = retirement.get_gate() + body_entered, body_finish = asyncio.Event(), asyncio.Event() + cleanup_entered, cleanup_finish = asyncio.Event(), asyncio.Event() + + async def body(): + yield b"first" + body_entered.set() + await body_finish.wait() + yield b"last" + + async def cleanup(): + cleanup_entered.set() + await cleanup_finish.wait() + + async def handler(): + return StreamingResponse(body(), background=BackgroundTask(cleanup)) + + app = _request_app(handler) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await body_entered.wait() + assert gate.active == 2 + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + body_finish.set() + await cleanup_entered.wait() + assert gate.active == 2 and not task.done() + assert not await barrier.retire_process() + cleanup_finish.set() + assert (await task).content == b"firstlast" + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_websocket_lifetime_and_shutdown_refusal_share_the_same_counter(): + import asyncio + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + return {} + + from starlette.websockets import WebSocket + + async def websocket(ws: WebSocket): + await ws.accept() + entered.set() + await finish.wait() + await ws.close() + + app = _request_app(handler) + app.add_api_websocket_route("/socket", websocket) + scope = { + "type": "websocket", + "path": "/socket", + "scheme": "ws", + "query_string": b"", + "headers": [(b"host", b"localhost")], + "subprotocols": [], + "client": ("127.0.0.1", 1234), + "server": ("localhost", 80), + } + task = asyncio.create_task( + app(scope, AsyncMock(return_value={"type": "websocket.connect"}), AsyncMock()) + ) + await entered.wait() + assert gate.active == 2 + gate.close() + send = AsyncMock() + await app(scope, AsyncMock(), send) + send.assert_awaited_once_with({"type": "websocket.close", "code": 1013}) + assert gate.active == 2 + finish.set() + await task + assert gate.active == 0 diff --git a/tests/unit/test_provider_barrier.py b/tests/unit/test_provider_barrier.py index 8cac6eceb8..551a8c998f 100644 --- a/tests/unit/test_provider_barrier.py +++ b/tests/unit/test_provider_barrier.py @@ -29,6 +29,11 @@ def _row(pid, transition=None, state="running", *, version=1, expired=False): @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().install("test") + retirement.get_gate().enrolled = True + retirement.get_gate().producers_stopped = True identity, transition = uuid4(), uuid4() gate = ModelWorkGate() monkeypatch.setattr(barrier, "get_model_work_gate", lambda: gate) @@ -37,6 +42,7 @@ def setup(monkeypatch): monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) record = AsyncMock() monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) rows = [_row(identity, transition, "drained")] monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) @@ -188,7 +194,7 @@ def test_a_heartbeat_carries_the_latest_state_above_any_version_seen(setup): "state": "drained", "version": 10**20 + 1, "stale_after_s": barrier.STALE_AFTER_S, - "host_proof": barrier.host_proof(), + "host_proof": barrier.participant_proof(), } # An acknowledgement stamped after the beat outranks it. assert barrier._OWN.stamp(transition, "drained")[2] > row[3] @@ -205,12 +211,14 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m await barrier.register_process() assert beat.running and len(ch.inserts) == 1 tags: list[str] = [] - record.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + barrier.record_retirement.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + beat.signal_stop() + beat.thread.join(timeout=5) assert await barrier.retire_process() assert beat.stopping.is_set() # signalled, not joined: the thread ends after its beat beat.thread.join(timeout=5) assert not beat.running - assert record.call_args.args[2] == "retired" + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() assert tags == ["provider.participant_heartbeat"] and current_query_tag() == "" beat.ch = ch beat.beat() @@ -218,7 +226,7 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m @pytest.mark.asyncio -async def test_a_boot_that_fails_after_enrolling_retires_its_row(setup, monkeypatch): +async def test_a_boot_that_fails_does_not_certify_a_pending_heartbeat(setup, monkeypatch): _, _, _, record = setup monkeypatch.setattr(barrier, "_HEARTBEAT", barrier._Heartbeat(lambda: _SyncCH())) monkeypatch.setattr(barrier, "_registered", False) @@ -232,7 +240,10 @@ async def lifespan(_app): with pytest.raises(RuntimeError, match="migrations failed"): async with barrier.participant_lifespan(lifespan)(None): pytest.fail("a failed boot never serves") - assert record.call_args.args[2] == "retired" + barrier.record_retirement.assert_not_awaited() + barrier._HEARTBEAT.thread.join(timeout=5) + assert await barrier.retire_process() + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() @pytest.mark.asyncio @@ -263,8 +274,9 @@ async def test_retire_participant_refuses_a_live_one(setup, monkeypatch): assert await barrier.retire_participant(other) == "live" record.assert_not_awaited() monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) - assert await barrier.retire_participant(other) == "retired" - record.assert_awaited_once_with(other, UUID(int=0), "retired", after=5, ch=None) + assert await barrier.retire_participant(other) == "live" + record.assert_not_awaited() + barrier.record_retirement.assert_not_awaited() def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): @@ -281,7 +293,7 @@ def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): monkeypatch.setattr(_ch, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = argparse.Namespace(participant=str(other), yes=True) assert participant._cmd_retire(args) == 1 - assert "still running" in capsys.readouterr().err + assert "no verifiable exact physical-death proof" in capsys.readouterr().err record.assert_not_awaited() @@ -434,14 +446,13 @@ async def _hang(*_a, **_k): @pytest.mark.asyncio -async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(): +async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(monkeypatch): ch = SimpleNamespace(insert=AsyncMock()) - await pp.record_participant(uuid4(), UUID(int=0), "running", ch=ch) + await pp.record_participant(pp.process_identity(), UUID(int=0), "running", ch=ch) (row,) = ch.insert.call_args.args[1] assert row["stale_after_s"] == pp.STALE_AFTER_S > barrier.HEARTBEAT_S - await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) - (row,) = ch.insert.call_args.args[1] - assert row["stale_after_s"] == 0 + with pytest.raises(ValueError, match="state"): + await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) @pytest.mark.asyncio diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 347506b522..c6ba785ab4 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -10,6 +10,9 @@ @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().enrolled = True gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 60cead66a5..d1ee7960e3 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -70,7 +70,7 @@ def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) - assert pp.proves_stopped(_with(container="c" * 64, pidns="1")) + assert not pp.proves_stopped(_with(container="c" * 64, pidns="1")) @pytest.mark.parametrize("theirs", ["d" * 64, ""]) @@ -152,127 +152,11 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) ) record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", record) monkeypatch.setattr(barrier, "_last_here", -float("inf")) assert await barrier.retire_dead_here(now=100.0) == 1 - record.assert_awaited_once_with(dead, UUID(int=0), "retired", after=5) + record.assert_awaited_once() + proof = record.await_args.args[0] + assert proof.participant == dead and proof.after == 5 + assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled - - -# -- a pod Kubernetes deleted ------------------------------------------------ - -_POD_UID = "0b6f2c1e-1111-2222-3333-444455556666" - - -def _pod_proof( - namespace: str = "loom", container: str = f"{_POD_UID}/loom", pod: str = "loom-7d9-abc" -) -> str: - # The host name is the pod name cut to 63 characters; the lookup uses the pod name. - return "|".join((pod[:63], "boot", container, "4026", "7", "99", namespace, pod)) - - -@pytest.fixture -def kube(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> Any: - """A service-account mount, the API's address and a stand-in API: it answers - the (status, body) pairs queued in `answers` and keeps what it saw in `seen`.""" - from types import SimpleNamespace - - import httpx - - (tmp_path / "token").write_text("tok") - (tmp_path / "ca.crt").write_text("") - monkeypatch.setattr(pp, "_SA", str(tmp_path)) - monkeypatch.setenv("POD_NAMESPACE", "loom") - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") - api = SimpleNamespace(answers=[], seen=[]) - - def handler(request: httpx.Request) -> httpx.Response: - api.seen.append(request) - status, body = api.answers.pop(0) - return httpx.Response(status, json=body) - - real = httpx.AsyncClient - monkeypatch.setattr( - httpx, - "AsyncClient", - lambda **kw: real(transport=httpx.MockTransport(handler), timeout=kw.get("timeout")), - ) - return api - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "status,uid,phase,gone", - [ - (404, "", "", True), - (200, "another-uid", "Running", True), - (200, _POD_UID, "Failed", True), # evicted: the object stays, its containers stopped - (200, _POD_UID, "Succeeded", True), - (200, _POD_UID, "Running", False), - (200, _POD_UID, "Pending", False), - (403, "", "", False), - ], -) -async def test_a_pod_is_gone_only_when_the_api_says_so(kube, status, uid, phase, gone) -> None: - kube.answers.append((status, {"metadata": {"uid": uid}, "status": {"phase": phase}})) - assert await pp.pod_gone(_pod_proof()) is gone - assert kube.seen[0].url.path == "/api/v1/namespaces/loom/pods/loom-7d9-abc" - assert kube.seen[0].headers["authorization"] == "Bearer tok" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "proof", - [ - _pod_proof(namespace="other"), - _pod_proof(container="a" * 64), - _pod_proof(pod=""), - "h|b|c|p|1|2", - "h|b|" + _POD_UID + "/loom|p|1|2|loom", # a 7-part proof (no pod name) proves nothing - "", - ], -) -async def test_a_pod_elsewhere_or_outside_kubernetes_is_never_gone(kube, proof) -> None: - assert await pp.pod_gone(proof) is False - assert kube.seen == [] # nothing to ask: another namespace, a Docker container, an old row - - -@pytest.mark.asyncio -async def test_the_leader_retires_an_expired_participant_whose_pod_is_gone( - monkeypatch: pytest.MonkeyPatch, -) -> None: - me, lost, busy, kept = (uuid4() for _ in range(4)) - monkeypatch.setattr(barrier, "process_identity", lambda: me) - rows = [ - _row(me, pp.host_proof()), - _row(lost, _pod_proof(), expired=True, version=9), - _row(busy, _pod_proof(), expired=True), - _row(kept, _pod_proof(), expired=True), - ] - monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) - monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) - monkeypatch.setattr( - barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) - ) - asked: list[str] = [] - - async def gone(proof: str) -> bool: - asked.append(proof) - return len(asked) == 1 # the first asked (lost) is gone, kept's pod still exists - - monkeypatch.setattr(barrier, "pod_gone", gone) - record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) - monkeypatch.setattr(barrier, "_STALE", None) - await barrier.sweep_stopped(now=5000.0) - record.assert_awaited_once_with(lost, UUID(int=0), "retired", after=9) - assert len(asked) == 2 # a participant with a query running is never asked about - assert [p["participant_id"] for p in barrier.stale_participants()] == [str(busy), str(kept)] - - -@pytest.mark.asyncio -async def test_a_long_pod_name_is_looked_up_whole_not_by_its_cut_host_name(kube) -> None: - long = "loom-memory-platform-production-workers-7d9c8b6f5d-x4kqz-abcdefgh" - kube.answers.append((200, {"metadata": {"uid": _POD_UID}, "status": {"phase": "Running"}})) - assert await pp.pod_gone(_pod_proof(pod=long)) is False - assert kube.seen[0].url.path == f"/api/v1/namespaces/loom/pods/{long}" diff --git a/tests/unit/test_reextract.py b/tests/unit/test_reextract.py index 852f94d1b0..addaef8003 100644 --- a/tests/unit/test_reextract.py +++ b/tests/unit/test_reextract.py @@ -159,7 +159,9 @@ def test_cli_refuses_the_source_as_target(capsys) -> None: assert "must differ" in capsys.readouterr().err -def test_reextract_token_falls_back_to_the_saved_token(tmp_path, monkeypatch) -> None: +def test_reextract_token_falls_back_to_the_saved_token( + tmp_path, monkeypatch, stub_model_command +) -> None: """--token, else LOOM_API_KEY, else ~/.loom/token: the order every client command uses.""" import argparse diff --git a/tests/unit/test_revocation_web_refresh.py b/tests/unit/test_revocation_web_refresh.py index bbc0f56a5a..8dac8ed351 100644 --- a/tests/unit/test_revocation_web_refresh.py +++ b/tests/unit/test_revocation_web_refresh.py @@ -133,16 +133,28 @@ def _poll_loop(period: float) -> asyncio.Task[None]: t.cancel() -def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown() -> None: - src = (ROOT / "server" / "main.py").read_text() - lifespan = src[src.index("async def lifespan(") :] - assert "auth_loops = await boot.start_token_state()" in lifespan - assert ( - "for loop in (*cell_loops, *auth_loops, overrides_loop):\n loop.cancel()" - in lifespan +@pytest.mark.asyncio +async def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown(monkeypatch) -> None: + import ast + + from server import boot, main + from server.code.app.index import mirrors + from server.core import retirement + from server.memory.app import embedder_calibration + + tree = ast.parse((ROOT / "server" / "main.py").read_text()) + lifespan = next( + n for n in ast.walk(tree) if isinstance(n, ast.AsyncFunctionDef) and n.name == "lifespan" ) - # the decorator still sits on lifespan, not on the helper above it - assert "@asynccontextmanager\nasync def lifespan(" in src + calls = {ast.unparse(n.func) for n in ast.walk(lifespan) if isinstance(n, ast.Call)} + assert {"boot.start_token_state", "_stop_background"} <= calls + monkeypatch.setattr(mirrors, "stop_warmup", lambda: None) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: None) + monkeypatch.setattr(boot, "cancel_boot_tasks", lambda: None) + task = retirement.get_gate().track(asyncio.create_task(asyncio.Event().wait())) + await main._stop_background(None) + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and retirement.get_gate().closed @pytest.mark.asyncio diff --git a/tests/unit/test_shutdown_deadline.py b/tests/unit/test_shutdown_deadline.py index e8d53b92f2..8f60ab0419 100644 --- a/tests/unit/test_shutdown_deadline.py +++ b/tests/unit/test_shutdown_deadline.py @@ -125,7 +125,7 @@ def close(self): pass # The parent measures from here to the process's exit, imports left out # (CLOCK_MONOTONIC is one clock for every process on the host). print("retiring", time.monotonic(), flush=True) -assert asyncio.run(barrier.retire_process()) +assert not asyncio.run(barrier.retire_process()) """ From 8a76924217c43166af9d5d4836cdfc6e5b4f5579 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:52:01 +0000 Subject: [PATCH 055/181] Hold native principal work through command retirement Keep claim lookup and release in one enrolled async body. Run token readiness, keyring loading, signing and audit before the sole command drain and retirement. Refuse native offline minting when paired-store verification or enrollment fails; a keyring-only read failure inside enrolled work can still use the root key. Remove unreachable registration-row branches and duplicate worker task tracking, and name the changed-namespace death regression after its fail-closed contract. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 + docs/guides/tokens-and-grants.md | 7 +- docs/reference/cli.md | 7 +- server/auth/keyring.py | 23 +- server/cli/_ch.py | 4 +- server/cli/access.py | 153 +++++++------ .../insights/store/provider_participants.py | 6 +- server/worker_main.py | 2 - tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_cli_orgs.py | 21 ++ tests/unit/test_cli_hygiene.py | 14 +- tests/unit/test_native_command_retirement.py | 201 ++++++++++++++++++ tests/unit/test_platform_admin_bots.py | 20 +- tests/unit/test_platform_admin_rollout.py | 8 +- tests/unit/test_provider_host_proof.py | 2 +- tests/unit/test_revoke_issued_before.py | 9 +- 17 files changed, 395 insertions(+), 97 deletions(-) create mode 100644 tests/unit/test_native_command_retirement.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 9cd44ad6f4..7fccee817d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -25,6 +25,17 @@ enrolls before its first fresh primary acceptance read and uses permanent guarde retirement. A serving cell heartbeat includes its `participant_id` beside the fingerprint, allowing a replacement to be joined to the participant registry. Workers children enroll independently and do not publish serving instance rows. +Native commands hold one admission scope through their complete org or principal +work and drain: claim lookup and release share that scope; token readiness, keyring +read, signing and audit share another. Connections and helper calls do not retire +a command that still has principal work to perform. Store pairing is verified +before enrollment and fresh acceptance. Org-free diagnostics do not install an +accepting view. The pre-schema `ch-setup` ownership probe discards its diagnostic +mode result and checks only store/cell identity before raw database user/profile +setup; its optional analyst-row work opens a separate verified, enrolled scope +and obtains fresh acceptance. That bootstrap exception admits no tenant or token +work. + The capability identifies installed code; it does not prove that every older process or unsafe retirement client has disappeared. diff --git a/docs/guides/tokens-and-grants.md b/docs/guides/tokens-and-grants.md index a5bf21c44a..63c919f0d7 100644 --- a/docs/guides/tokens-and-grants.md +++ b/docs/guides/tokens-and-grants.md @@ -26,7 +26,12 @@ loom mint-token --subject my-bot --org acme --namespaces support --roles writer `loom` ships with the server package and the Docker image, not with the SDK; in a container run `docker exec loom mint-token ...`. Minting needs the signing key, -`LOOM_JWT_SECRET`. A fresh clone gets a development secret from `.env.example`. +`LOOM_JWT_SECRET`, and access to the deployment's migrated stores. The command verifies +the paired stores and enrolls before reading the accepting cell state or signing. +Offline native minting and minting before migrations refuse. If only the keyring read +fails after successful enrollment, an ordinary mint can still sign with the root key. +Platform-admin minting also requires a fresh readable fleet and a recorded audit event. A fresh clone gets a development +secret from `.env.example`. `loom mint-token` and `loom doctor` warn while it is in use, and a deployment with auth on refuses to start with it (except the `make dev` stack). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3e0b9068ea..aa4218fbde 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -38,7 +38,10 @@ Commands fall into two kinds: whatever `--token` says, `loom init` refuses before it signs anything up, and `loom call` exchanges the workload token itself. -`loom mint-token` needs only `LOOM_JWT_SECRET`: it signs a token locally. +`loom mint-token` signs locally with `LOOM_JWT_SECRET` after verifying its paired +stores and enrolling as a participant. It refuses when those stores cannot be verified +or enrollment fails, including offline and before the deployment is migrated. Run it +against the deployment's migrated stores. **Session commands** (`login`, `logout`, `whoami`, `call`, `token`) are the Python SDK's own (`loom.cli`; `python -m loom …` with only the SDK installed), so `loom` @@ -304,7 +307,7 @@ python scripts/codegen/docs_reference.py | `loom import-recall` | Import an external vector-recall agent-memory SQLite export into Loom. | `--sqlite`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom import-brain` | Import a markdown agent 'brain' directory into Loom: MEMORY.md (always-injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on-demand WARM layer) → knowledge.add docs. | `--dir`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom claim-release` | Release a self-signup claim: revoke its grants, clear the owner, deactivate it. | `--org`, `--namespace`, `-y` | -| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | +| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | | `loom revoke-token` | Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). | `--jti`, `--subject`, `--list`, `--issued-before`, `--scim`, `--reason`, `--expires-at`, `--json` | | `loom sso` | Carry password accounts over to single sign-on. | | | `loom sso adopt` | Let SSO sign-ins reach the password accounts whose email is at --domain: each person then signs in with the identity provider to their own account, its namespaces and data. | `--domain`, `--org`, `--skip`, `--keep-tokens`, `--apply`, `--dry-run` | diff --git a/server/auth/keyring.py b/server/auth/keyring.py index d4624a4369..ff5c413d4b 100644 --- a/server/auth/keyring.py +++ b/server/auth/keyring.py @@ -162,8 +162,9 @@ def sign_with_root_only() -> None: def load_for_offline_mint(*, run: Callable[..., Any] | None = None) -> bool: """For a process that mints without serving (a CLI, a dev or bench script): read the keyring once, or sign as kid `default` when ClickHouse - cannot be read. `run` is the CLI's `run_connected`; without it the read - uses a client of its own. True when it was read. Raises ValueError when + cannot be read. `run` can supply a connected read; without it the read + uses a client of its own. Native CLI minting uses `load_for_connected_mint` + inside its enrolled command instead. True when it was read. Raises ValueError when the keyring was recorded under another root secret: no server would accept what this process signs; through `run`, raises AuthorityMismatch when the shell is configured for another deployment (`connect_checked`).""" @@ -191,6 +192,24 @@ async def _read() -> bool: raise # a shell configured for another deployment: refused, not signed by root except Exception: # noqa: BLE001 — an offline mint signs with the root secret read = False + return _offline_mint_result(read) + + +async def load_for_connected_mint(*, ch: Any) -> bool: + """Read signing keys inside an already verified, enrolled native command.""" + from server.core.pg import AuthorityMismatch + from server.core.retirement import ProcessStopping + + try: + read = await refresh(ch=ch) + except (AuthorityMismatch, ProcessStopping): + raise + except Exception: # noqa: BLE001 — an enrolled mint can sign with the root secret + read = False + return _offline_mint_result(read) + + +def _offline_mint_result(read: bool) -> bool: if not read: print(" · signing keyring not read; signing as kid 'default'", file=sys.stderr) sign_with_root_only() diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 184aa1f6eb..c67479b965 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -193,7 +193,9 @@ async def check_setup_stores() -> None: gated connect (`loom ch-setup`), the gate a gated connect would run, before it writes: its ClickHouse, where it already records a deployment, must record its Postgres's, and the Postgres `first` row must not name another cell. A ClickHouse with no Loom - database or record yet (a service set up before the first boot) passes.""" + database or record yet (a service set up before the first boot) passes. This + diagnostic discards the mode result and never installs an accepting view; + subsequent Loom row work must verify, enroll and check acceptance afresh.""" from server.core import pg, pg_profile from server.core.ch import get_client from server.core.ch_errors import error_code, error_line diff --git a/server/cli/access.py b/server/cli/access.py index 25cfc05667..482aea7b1a 100644 --- a/server/cli/access.py +++ b/server/cli/access.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json import sys from datetime import UTC @@ -39,31 +40,34 @@ async def _lookup_owner(ch: Any) -> str | None: return None return str(row.get("owner_email") or "") or "(unknown)" - owner = run_connected(_lookup_owner, org_free=reads("reads")) - if owner is None: - print( - f"no claim found for {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - if not args.yes: - print( - f"This will release the claim on {args.org}/{args.namespace}" - f" (currently owned by {owner!r})." - ) - print( - "Grants are revoked and the namespace deactivated. Its memories are " - "kept, and the name stays reserved: nobody can claim it again." - ) - try: - resp = input("Continue? [y/N] ").strip().lower() - except EOFError: # no terminal to answer (a pipe, a cron): the answer is no - resp = "" - if resp not in ("y", "yes"): - print("aborted") - return 1 - async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + owner = await _lookup_owner(ch) + if owner is None: + print( + f"no claim found for {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + if not args.yes: + print( + f"This will release the claim on {args.org}/{args.namespace}" + f" (currently owned by {owner!r})." + ) + print( + "Grants are revoked and the namespace deactivated. Its memories are " + "kept, and the name stays reserved: nobody can claim it again." + ) + try: + resp = (await asyncio.to_thread(input, "Continue? [y/N] ")).strip().lower() + except EOFError: # no terminal to answer (a pipe, a cron): the answer is no + resp = "" + if resp not in ("y", "yes"): + print("aborted") + return 1 + # Confirmation can wait while the cell changes; check again before mutation. + await check_acceptance(orgs=(args.org,)) out = await release_claim( org=args.org, namespace=args.namespace, @@ -110,8 +114,6 @@ def _invalid_namespace_names(names: list[str]) -> list[str]: def _cmd_mint_token(args: argparse.Namespace) -> int: from server.auth import keyring - from server.auth.authz import GRANT_ROLES - from server.auth.jwt import mint_token, new_jti from server.core.principals import RESERVED_SUBJECT, is_reserved_subject if is_reserved_subject(args.subject): @@ -122,19 +124,49 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: if refused: print(f" ✗ {refused}", file=sys.stderr) return 2 - if args.platform_admin and not _platform_admin_rollout_ready(): + + async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + if args.platform_admin and not await _platform_admin_rollout_ready(ch): + print( + " ✗ platform-admin tokens require a readable fleet with the rollout complete", + file=sys.stderr, + ) + return 1 + try: + await keyring.load_for_connected_mint(ch=ch) + except ValueError as e: # another root: no server would accept the token + print(f" ✗ {e}", file=sys.stderr) + return 1 + # Keyring reads can wait; signing must use a current accepting view too. + await check_acceptance(orgs=(args.org,)) + minted = _mint_token(args) + if isinstance(minted, int): + return minted + token, summary = minted + if args.platform_admin and not await _audit_platform_admin_mint(args, summary): + print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) + return 1 + return _emit_minted_token(args, token, summary) + + try: + return run_connected(_go, orgs=(args.org,)) + except AuthorityMismatch: + raise + except Exception as e: # noqa: BLE001 — a mint without verified enrollment is refused print( - " ✗ platform-admin tokens require a readable fleet with the rollout complete", + f" ✗ token mint did not complete in its enrolled scope: {type(e).__name__}", file=sys.stderr, ) return 1 - try: - keyring.load_for_offline_mint( - run=lambda work: run_connected(work, org_free=reads("reads the signing keyring")) - ) - except ValueError as e: # another root: no server would accept the token - print(f" ✗ {e}", file=sys.stderr) - return 1 + + +def _mint_token(args: argparse.Namespace) -> tuple[str, dict[str, Any]] | int: + """Build and sign only inside the command's enrolled admission scope.""" + from server.auth.authz import GRANT_ROLES + from server.auth.jwt import mint_token, new_jti + ttl_seconds = (args.ttl_days * 86400) if args.ttl_days else None # Every minted token carries a jti; generated here so # the summary can name it beside (or instead of) the token — it is the @@ -238,11 +270,7 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: "consumer": args.consumer or "", } - if args.platform_admin and not _audit_platform_admin_mint(args, summary): - # Never handed out unrecorded: the role spans every org. - print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) - return 1 - return _emit_minted_token(args, token, summary) + return token, summary # not a setting: the longest a minted platform-admin token may live; longer is a bot's job @@ -272,45 +300,40 @@ def _platform_admin_refusal(args: argparse.Namespace) -> str | None: return None -def _platform_admin_rollout_ready() -> bool: +async def _platform_admin_rollout_ready(ch: Any) -> bool: from server.auth import machines try: - return bool( - run_connected( - machines.platform_admin_tokens_ready, org_free=reads("reads the live instances") - ) - ) + return bool(await machines.platform_admin_tokens_ready(ch)) except AuthorityMismatch: - raise # a shell configured for another deployment: its own one-line refusal + raise except Exception as e: # noqa: BLE001 — an unread fleet refuses an admin token print(f" ✗ fleet state could not be read: {type(e).__name__}", file=sys.stderr) return False -def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: +async def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: """Record a platform-admin mint before it is handed out; False when the audit row could not be written.""" import getpass from server.observability import audit - async def _emit(ch: Any) -> str: - return await audit.emit( - "auth.platform_admin_minted", - org=args.org, - namespace="-", - actor=f"cli:{getpass.getuser()}", - subject={ - "subject": args.subject, - "jti": summary["jti"], - "namespaces": summary["namespaces"], - "ttl_days": args.ttl_days, - }, - ) - try: - return bool(run_connected(_emit, orgs=(args.org,))) + return bool( + await audit.emit( + "auth.platform_admin_minted", + org=args.org, + namespace="-", + actor=f"cli:{getpass.getuser()}", + subject={ + "subject": args.subject, + "jti": summary["jti"], + "namespaces": summary["namespaces"], + "ttl_days": args.ttl_days, + }, + ) + ) except Exception as e: # noqa: BLE001 — an unrecorded mint is refused, never handed out print(f" ✗ audit log not reached: {type(e).__name__}: {e}", file=sys.stderr) return False @@ -714,7 +737,9 @@ def _add_revoke_token_parser(sub: Any) -> None: def _add_mint_token_parser(sub: Any) -> None: """The `mint-token` subcommand.""" - p_mint = sub.add_parser("mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET).") + p_mint = sub.add_parser( + "mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores)." + ) p_mint.add_argument("--subject", required=True) p_mint.add_argument("--org", required=True) p_mint.add_argument( diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index da04df218d..faa8bc1d54 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -111,10 +111,8 @@ async def record_participant( "transition_id": transition, "state": state, "version": version if version is not None else max(time_ns(), after + 1), - # A retired row is stopped at once, so every build's `provider-lock clear` - # frees its holder without waiting out the heartbeat window. - "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": participant_proof() if own else "", + "stale_after_s": STALE_AFTER_S, + "host_proof": participant_proof(), } ], ) diff --git a/server/worker_main.py b/server/worker_main.py index c265c7c124..4a606a4291 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -63,8 +63,6 @@ async def _trackers() -> None: track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) - for task in tasks: - retirement.get_gate().track(task) return tasks diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 37e0e5e39e..f0f3075ad1 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -2,7 +2,7 @@ "loom": [ "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...", "Loom — vendor-neutral agent memory plane on ClickHouse.", - "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", + "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", "options: -h, --help show this help message and exit --version show program's version number and exit" ], "loom code": [ diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 8216d4de36..d689f7fdf6 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -15607,7 +15607,7 @@ }, { "dest": "mint-token", - "help": "Mint a JWT locally (requires LOOM_JWT_SECRET).", + "help": "Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores).", "metavar": "mint-token" }, { diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 6d14350e03..2e1fc087ee 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -475,6 +475,27 @@ async def connect(self) -> None: assert not [r for r in caplog.records if r.name == "clickhouse_connect"] +@pytest.mark.parametrize("diagnostic_mode", ["open", "recording", "enforced", "partial"]) +def test_setup_identity_probe_neither_installs_nor_uses_acceptance( + monkeypatch: pytest.MonkeyPatch, diagnostic_mode: str +) -> None: + """Bootstrap checks store ownership, never a reusable data-plane admission.""" + from server.core import retirement + + _cell_set(monkeypatch, (diagnostic_mode, "eu-1")) + client = AsyncMock() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "recorded_authority", AsyncMock(return_value=None)) + acceptance = AsyncMock(side_effect=AssertionError("bootstrap is not admission")) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(cell, "refresh", acceptance) + before = cell._view + asyncio.run(_ch.check_setup_stores()) + assert cell._view is before + assert not retirement.get_gate().enrolled + acceptance.assert_not_awaited() + + def test_the_loom_code_drain_after_a_subcommand_is_a_read() -> None: """The reconnect that drains buffered writes runs after every `loom code` subcommand, reads included; the subcommand's own connect gated its writes, so the diff --git a/tests/unit/test_cli_hygiene.py b/tests/unit/test_cli_hygiene.py index 5458194131..630ca7d2de 100644 --- a/tests/unit/test_cli_hygiene.py +++ b/tests/unit/test_cli_hygiene.py @@ -171,7 +171,7 @@ def test_mint_token_signs_with_the_keyring_it_reads( pyjwt.decode(token, "k" * 40, algorithms=["HS256"], audience=settings.jwt_audience) -def test_an_offline_mint_signs_with_the_root_secret( +def test_an_offline_native_mint_refuses_without_enrollment( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _auth_on_with(monkeypatch, "a-real-secret-that-is-not-the-default") @@ -180,10 +180,9 @@ def _unreachable(run: Any, **_declared: object) -> None: raise ConnectionError("no clickhouse") monkeypatch.setattr(access, "run_connected", _unreachable) - assert cli.main(_MINT) == 0 + assert cli.main(_MINT) == 1 out, err = capsys.readouterr() - assert "signing keyring not read; signing as kid" in err - assert pyjwt.get_unverified_header(out.strip())["kid"] == "default" + assert out == "" and "enrolled scope" in err def _doctor(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> tuple[int, str]: @@ -324,8 +323,13 @@ def _refuse(_run: object, **_declared: object) -> bool: raise AuthorityMismatch("this deployment's memory authority is postgres") monkeypatch.setattr(access, "run_connected", _refuse) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers()) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--platform-admin", "--ttl-days", "1"] + ) with pytest.raises(AuthorityMismatch): - access._platform_admin_rollout_ready() + access._cmd_mint_token(args) def test_a_revocation_for_another_deployment_is_refused_as_such( diff --git a/tests/unit/test_native_command_retirement.py b/tests/unit/test_native_command_retirement.py new file mode 100644 index 0000000000..228352a83d --- /dev/null +++ b/tests/unit/test_native_command_retirement.py @@ -0,0 +1,201 @@ +"""The actual native command cannot outlive its certified participant.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch, access +from server.core import retirement +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server.core import background, ch, pg, pg_profile + from server.core.model_work_gate import ModelWorkGate + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + + async def connect(): + client.connected = True + + async def disconnect(): + client.connected = False + + client.connect, client.disconnect = connect, disconnect + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", AsyncMock()) + + async def verified(_): + assert not gate.enrolled + events.append("verified") + + async def enroll(): + assert events == ["verified"] + gate.enrolled = True + events.append("enrolled") + + async def acceptance(**_): + assert gate.enrolled and gate.active > 0 and not gate.closed + events.append("acceptance") + + async def certificate(proof): + assert gate.quiescent and client.connected + assert events[-1] == "buffer" + events.append("certificate") + + async def flush(): + events.append("buffer") + return True + + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + model_gate = ModelWorkGate() + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(command, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(barrier, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(command, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "record_retirement", certificate) + return gate, events + + +def test_actual_claim_release_keeps_lookup_and_release_in_one_lifetime(native, monkeypatch): + import asyncio + + from server.core.background import create_deferred_task + from server.identity.app import claim_release + from server.identity.store import namespaces + + gate, events = native + + async def lookup(*_, **__): + assert gate.active > 0 and not gate.closed + events.append("lookup") + return {"owner_email": "owner@example.test"} + + async def release(**_): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append("release") + + async def descendant(): + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("descendant") + + async def write(): + await asyncio.sleep(0) + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("deferred") + create_deferred_task(descendant()) + + create_deferred_task(write()) + return SimpleNamespace(grants_revoked=2) + + monkeypatch.setattr(namespaces, "fetch_row", lookup) + monkeypatch.setattr(claim_release, "release_claim", release) + assert access._cmd_claim_release(SimpleNamespace(org="o", namespace="n", yes=True)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("lookup") < events.index("release") < events.index("certificate") + assert events.index("release") < events.index("deferred") < events.index("descendant") + assert events.index("descendant") < events.index("buffer") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("platform_admin", [False, True]) +def test_actual_mint_readiness_keyring_signing_and_audit_precede_retirement( + native, monkeypatch, platform_admin +): + import argparse + + from server.auth import jwt, keyring, machines + from server.observability import audit + + gate, events = native + + def observed(event): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append(event) + + async def ready(_): + observed("readiness") + return True + + async def refresh(**_): + observed("keyring") + return True + + def mint(**_): + observed("mint") + return "test-token" + + async def record(*_, **__): + observed("audit") + return "audit-id" + + monkeypatch.setattr(machines, "platform_admin_tokens_ready", ready) + monkeypatch.setattr(keyring, "refresh", refresh) + monkeypatch.setattr(jwt, "mint_token", mint) + monkeypatch.setattr(audit, "emit", record) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + argv = ["mint-token", "--subject", "sa:test", "--org", "o", "--ttl-days", "1"] + argv += ["--platform-admin"] if platform_admin else ["--namespaces", "n"] + assert access._cmd_mint_token(parser.parse_args(argv)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("keyring") < events.index("mint") < events.index("certificate") + if platform_admin: + assert events.index("readiness") < events.index("keyring") + assert events.index("mint") < events.index("audit") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("failure", ["unavailable", "wrong-pair"]) +def test_actual_mint_never_signs_without_verified_enrollment(native, monkeypatch, failure): + import argparse + from unittest.mock import Mock + + from server.auth import jwt + from server.core import pg_profile + from server.core.pg import AuthorityMismatch + + gate, events = native + + async def refuse(_): + if failure == "wrong-pair": + raise AuthorityMismatch("different paired stores") + raise ConnectionError("unavailable") + + minted = Mock(side_effect=AssertionError("unenrolled mint")) + monkeypatch.setattr(pg_profile, "check_configured", refuse) + monkeypatch.setattr(jwt, "mint_token", minted) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--namespaces", "n"] + ) + if failure == "wrong-pair": + with pytest.raises(AuthorityMismatch): + access._cmd_mint_token(args) + else: + assert access._cmd_mint_token(args) == 1 + minted.assert_not_called() + assert not gate.enrolled and not gate.closed and events == [] diff --git a/tests/unit/test_platform_admin_bots.py b/tests/unit/test_platform_admin_bots.py index 84fd4c1834..ed2ddb14b2 100644 --- a/tests/unit/test_platform_admin_bots.py +++ b/tests/unit/test_platform_admin_bots.py @@ -13,6 +13,7 @@ import asyncio import json from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -181,13 +182,16 @@ def minting(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: monkeypatch.setenv("LOOM_JWT_SECRET", _SECRET) monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + from server.auth import keyring + + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) # These mint tests run after every replica has the human-admin guards. - monkeypatch.setattr(access, "_platform_admin_rollout_ready", lambda: True) + monkeypatch.setattr(access, "_platform_admin_rollout_ready", AsyncMock(return_value=True)) recorded: list[dict[str, Any]] = [] - def audited(args: Any, summary: dict[str, Any]) -> bool: + async def audited(args: Any, summary: dict[str, Any]) -> bool: recorded.append({"subject": args.subject, "jti": summary["jti"]}) return True @@ -236,7 +240,7 @@ def test_a_platform_admin_mint_the_audit_log_misses_is_not_handed_out( ) -> None: from server.cli import access - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda args, summary: False) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=False)) assert cli.main([*_MINT, "--subject", "sa:cp", "--ttl-days", "7"]) == 1 out, err = capsys.readouterr() assert out == "" and "no token" in err @@ -257,7 +261,7 @@ async def emit(event: str, **kw: Any) -> str: monkeypatch.setattr(audit, "emit", emit) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = SimpleNamespace(subject="sa:cp", org="acme", ttl_days=7) - assert access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]}) + assert asyncio.run(access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]})) [(event, kw)] = rows assert event == "auth.platform_admin_minted" and kw["org"] == "acme" assert kw["subject"] == {"subject": "sa:cp", "jti": "j1", "namespaces": ["*"], "ttl_days": 7} @@ -266,7 +270,9 @@ async def lost(event: str, **kw: Any) -> str: return "" # audit.emit's answer when its insert failed monkeypatch.setattr(audit, "emit", lost) - assert not access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + assert not asyncio.run( + access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + ) _CP_BODY = { diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 134f54cc8a..93d3a5eae5 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -4,6 +4,7 @@ import time from typing import Any +from unittest.mock import AsyncMock import pytest @@ -87,9 +88,10 @@ def mint(**kw: Any) -> str: monkeypatch.setattr(instances, "live_rows", lambda: [{"schema_version": 76}]) monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(keyring, "load_for_offline_mint", lambda **kw: None) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda *a: True) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) monkeypatch.setattr(jwt, "mint_token", mint) code = cli.main( [ @@ -139,7 +141,7 @@ def mint(**kw): ] ) out, err = capsys.readouterr() - assert code == 1 and out == "" and "rollout" in err + assert code == 1 and out == "" and "enrolled scope" in err def test_admin_token_floor_matches_the_physical_migration() -> None: diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index d1ee7960e3..0e70602dd7 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -66,7 +66,7 @@ def _eperm(pid: int, sig: int) -> None: assert not pp.proves_stopped(_with(pid=_dead_pid())) -def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( +def test_a_changed_container_pid_namespace_does_not_prove_the_old_process_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) diff --git a/tests/unit/test_revoke_issued_before.py b/tests/unit/test_revoke_issued_before.py index e7e9bb0af0..fa0c71444d 100644 --- a/tests/unit/test_revoke_issued_before.py +++ b/tests/unit/test_revoke_issued_before.py @@ -20,6 +20,7 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -265,10 +266,12 @@ def test_cli_mint_reports_the_jti_but_never_beside_a_saved_token( from server.settings import settings monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) + from server.auth import keyring from server.cli import access - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) rc = cli.main( ["mint-token", "--subject", "sa:x", "--org", "acme", "--namespaces", "shop", "--json"] ) @@ -539,6 +542,6 @@ def test_cli_mint_refuses_a_reserved_subject_without_a_traceback( def _read(**_k: Any) -> None: raise AssertionError("the keyring was read for a subject no token may carry") - monkeypatch.setattr(keyring, "load_for_offline_mint", _read) + monkeypatch.setattr(keyring, "load_for_connected_mint", _read) rc = cli.main(["mint-token", "--subject", subject, "--org", "acme", "--namespaces", "shop"]) assert rc == 2 and "grant principal" in capsys.readouterr().err From deacbaa8c36f2691d461669d1474e068dccec8a3 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:14:57 +0000 Subject: [PATCH 056/181] Close directory control sessions before cancelling their work Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 19 +- server/router/directory.py | 228 +++++++++++--- tests/integration/test_cell_control_fence.py | 6 +- .../test_directory_control_cleanup.py | 296 ++++++++++++++++++ tests/integration/test_router_directory_pg.py | 11 +- tests/unit/test_cell_attach_cleanup.py | 62 ++-- tests/unit/test_cell_control_census.py | 54 +++- tests/unit/test_directory_control_cleanup.py | 136 ++++++++ tests/unit/test_router_attach.py | 10 +- 10 files changed, 735 insertions(+), 88 deletions(-) create mode 100644 tests/integration/test_directory_control_cleanup.py create mode 100644 tests/unit/test_directory_control_cleanup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b5b7aced8..1abff33cbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -904,6 +904,7 @@ jobs: tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py + tests/integration/test_directory_control_cleanup.py tests/integration/test_cell_org_triggers_pg.py - name: pytest online-index durability on a private physical standby timeout-minutes: 8 diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c0a4115499..a0079ac900 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -990,10 +990,12 @@ advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (pre takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would refuse every later run, and attach's tenancy rows would be refused as an unchecked -writer's. A lock found lost between steps is taken again (refusing when another run holds -it), and the cell's state is read again. The lock does not make a run safe, since every -step is idempotent and the mode only rises; it keeps a second run's scans from holding the -snapshot horizon that step 4 waits on. +writer's. A lost or unreadable lock session ends the run: it never reconnects or +reacquires the lock. Rerun attach to obtain a fresh directory claim and recompute the +remaining work. Every local acceptance write uses this exact backend, so losing the +lock also ends its authority to change acceptance. The lock serializes prepare, attach +and update, and keeps concurrent scans from holding the snapshot horizon that step 4 +waits on. It refuses, before any write, naming each: @@ -1268,6 +1270,15 @@ that the same backend still holds ATTACH. A COMMIT, rollback or ownership-check never becomes a busy retry. After attempting the mode update, the diagnostic stays unknown until rollback plus ownership or COMMIT is acknowledged. +Directory control mutations supervise the complete callback transaction independently, +including acquisition and COMMIT. One 131-second bound includes public close, child +settlement and pool-context exit, with cleanup time reserved inside it. Both a failed +attempt and an acknowledged successful transaction close only their own acquired +connection before releasing the pool context. Unexpected errors never wait for rollback +on that discarded session. A late acquisition cannot start mutation work or obtain a +fresh cleanup allowance; acquisition without a known proxy remains an unknown outcome. +Ordinary live placement retains its separate transaction and connection handling. + The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: diff --git a/server/router/directory.py b/server/router/directory.py index 5158d24cf7..42c48ff1fa 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -20,6 +20,7 @@ import asyncio import hmac import json +import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping from contextlib import AsyncExitStack, asynccontextmanager from dataclasses import dataclass @@ -28,6 +29,7 @@ from uuid import UUID from server.core import cell as cell_state +from server.core import pg from server.core.paths import ROUTER_MIGRATIONS_DIR from server.router.limits import MAX_BODY_BYTES from server.settings import settings @@ -383,11 +385,13 @@ async def _operator_change( change: Callable[[Any, Cell], Awaitable[dict[str, Any]]], expected: Control, ) -> Cell: - async with _attach_change(cell_name, expected) as (conn, found): + async def change_locked(conn: Any, found: Attaching) -> Cell: detail = await change(conn, found.row) await _audit(conn, action, {"cell": cell_name, **detail}) row = await conn.fetchrow(f"{_CELL} WHERE cell = $1", cell_name) - return _cell(row) + return _cell(row) + + return await _attach_change(cell_name, expected, change_locked) @dataclass(frozen=True) @@ -898,34 +902,159 @@ def control_refusal(found: Attaching, expected: Control, *, claim: bool = False) return "" -@asynccontextmanager -async def _control_transaction() -> AsyncIterator[Any]: - """Bound acquisition, statements, COMMIT and release; never await rollback on cancel.""" - async with _reading(), asyncio.timeout(CONTROL_MAX_S), _connection() as conn: - settled = False - tx = conn.transaction() +@dataclass(frozen=True) +class _ControlFatal: + error: SystemExit | KeyboardInterrupt + + +async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFatal: + try: + return await work() + except (SystemExit, KeyboardInterrupt) as error: + return _ControlFatal(error) + except _ControlInterrupted as error: + return error.fatal + + +@dataclass(frozen=True) +class _ControlConnection: + conn: Any + + async def close(self, **kwargs: Any) -> None: try: - await tx.start() + await self.conn.close(**kwargs) + except (SystemExit, KeyboardInterrupt) as error: + # pg.close_owned creates its own public-close child. + raise _ControlInterrupted(_ControlFatal(error)) from None + + def terminate(self) -> None: + pg._terminate(self.conn) + + +class _ControlOwner: + """Own only this operator attempt's proxy, through close and pool release.""" + + def __init__(self) -> None: + self.end = time.monotonic() + CONTROL_MAX_S + self.work_end = self.end - min(1.0, CONTROL_MAX_S / 4) + self.context = _connection() + self.conn: Any = None + self.entered = False + self.closing = False + self.close_task: asyncio.Task[Any] | None = None + self.exit_task: asyncio.Task[Any] | None = None + + def start[T](self, work: Callable[[], Awaitable[T]]) -> asyncio.Task[T | _ControlFatal]: + task = asyncio.create_task(_control_result(work)) + task.add_done_callback(pg._retrieve_task) + return task + + async def wait[T](self, task: asyncio.Task[T | _ControlFatal], end: float) -> T: + done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("directory control deadline expired; outcome unknown") + result = task.result() + if isinstance(result, _ControlFatal): + # Keep fatal exceptions inert in nested acquisition/cleanup children. + raise _ControlInterrupted(result) + return result + + async def close(self) -> None: + if self.conn is None: + return + if self.close_task is None: + self.close_task = self.start( + lambda: pg.close_owned(_ControlConnection(self.conn), end=self.end) + ) + try: + await self.wait(self.close_task, self.end) + except BaseException: + pg._terminate(self.conn) + self.close_task.cancel() + raise + + async def release(self) -> None: + if self.entered: + if self.exit_task is None: + self.exit_task = self.start(lambda: self.context.__aexit__(None, None, None)) try: - yield conn - except Exception: - await tx.rollback() - settled = True + await self.wait(self.exit_task, self.end) + except BaseException: + self.exit_task.cancel() raise - else: - await tx.commit() - settled = True - finally: - if not settled: - # A cancelled/unknown COMMIT cannot return this backend to the pool. - conn.terminate() + + async def work[T](self, change: Callable[[Any], Awaitable[T]]) -> T: + self.conn = await self.context.__aenter__() + self.entered = True + if self.closing: + # A cancellation-resistant acquire cannot hand a late proxy to a mutation. + try: + await self.close() + finally: + await self.release() + raise TimeoutError("directory connection arrived after its attempt ended") + tx = self.conn.transaction() + await tx.start() + result = await change(self.conn) + await tx.commit() + return result + + async def finish(self, task: asyncio.Task[Any]) -> None: + self.closing = True + failure: BaseException | None = None + try: + await self.close() + except BaseException as error: # noqa: BLE001 — release must preserve the first failure. + failure = error + if not task.done(): + task.cancel() + try: + await asyncio.wait({task}, timeout=max(0, self.end - time.monotonic())) + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + try: + await self.release() + except BaseException as error: # noqa: BLE001 — cleanup keeps the first failure. + failure = failure or error + if failure is not None: + raise failure -@asynccontextmanager -async def _attach_change( - cell_name: str, expected: Control, *, claim: bool = False -) -> AsyncIterator[tuple[Any, Attaching]]: - async with _control_transaction() as conn: +class _ControlInterrupted(BaseException): + def __init__(self, fatal: _ControlFatal) -> None: + self.fatal = fatal + + +async def _control_transaction[T](change: Callable[[Any], Awaitable[T]]) -> T: + """Bound the whole callback, close first, then settle work and pool ownership.""" + async with _reading(): + owner = _ControlOwner() + task = owner.start(lambda: owner.work(change)) + try: + result = await owner.wait(task, owner.work_end) + except BaseException as original: + try: + await owner.finish(task) + except BaseException: # noqa: BLE001 — preserve the original failure/cancellation. + pass + if isinstance(original, _ControlInterrupted): + raise original.fatal.error from None + raise + try: + await owner.finish(task) + except _ControlInterrupted as interrupted: + raise interrupted.fatal.error from None + return result + + +async def _attach_change[T]( + cell_name: str, + expected: Control, + change: Callable[[Any, Attaching], Awaitable[T]], + *, + claim: bool = False, +) -> T: + async def locked(conn: Any) -> T: await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) found = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1 FOR UPDATE", cell_name) if found is None: @@ -934,22 +1063,26 @@ async def _attach_change( attaching = _attaching(found, other) if reason := control_refusal(attaching, expected, claim=claim): raise OperatorRefused(reason) - # A no-op still has a synchronous WAL commit: a primary-visible earlier write - # whose acknowledgement was lost is not yet a durability proof. + # Even a no-op needs synchronous WAL acknowledgement. await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") - yield conn, attaching + return await change(conn, attaching) + + return await _control_transaction(locked) async def claim(cell_name: str, expected: Control) -> Attaching: """Rotate on EVERY execution, including an identical HTTP replay.""" - async with _attach_change(cell_name, expected, claim=True) as (conn, _found): + + async def change_locked(conn: Any, _found: Attaching) -> Attaching: await conn.execute( "UPDATE loom_router.cells SET mutation_generation = gen_random_uuid() WHERE cell = $1", cell_name, ) await _audit(conn, "control_claim", {"cell": cell_name}) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, _found.other) + return _attaching(written, _found.other) + + return await _attach_change(cell_name, expected, change_locked, claim=True) async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: @@ -964,7 +1097,8 @@ async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> At """`attach/begin`: write `attach_started_at` once, the fingerprints while the row has none, and the root-secret fingerprint on every call, audited with the allow lists. Idempotent: a rerun posting the same fingerprints writes only `keys_fp`.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := begin_refusal(found, begin): raise OperatorRefused(reason) await conn.execute( @@ -987,7 +1121,9 @@ async def attach_begin(cell_name: str, begin: Begin, *, expected: Control) -> At }, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) async def attach_place( @@ -1002,7 +1138,8 @@ async def attach_place( reads `enforced`, judges that.""" _refuse_long(orgs) batch = sorted(set(orgs)) - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> tuple[int, int]: if reason := attach_refusal(found, deployment_id, enforced=True): raise OperatorRefused(reason) new = await conn.fetch( @@ -1041,13 +1178,16 @@ async def attach_place( len(batch), len(new), ) - return len(new), int(placed) + return len(new), int(placed) + + return await _attach_change(cell_name, expected, change_locked) async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/enforced`: record directory mode `enforced` (registration wrote `recording`), after which the cell places new orgs live; idempotent.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: if reason := attach_refusal(found, deployment_id): raise OperatorRefused(reason) await conn.execute( @@ -1061,7 +1201,9 @@ async def attach_enforced(cell_name: str, deployment_id: str, *, expected: Contr {"cell": cell_name, "changed": found.row.mode != "enforced"}, ) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def issuers(row: Cell) -> list[str]: @@ -1079,7 +1221,8 @@ def issuers(row: Cell) -> list[str]: async def attach_serving(cell_name: str, deployment_id: str, *, expected: Control) -> Attaching: """`attach/serving`: mark the first cell `serving`, with the issuers its fingerprints accept computed from the row as locked. A `serving` row is answered as it is.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Attaching: row = found.row if found.served(deployment_id): await _audit(conn, "attach_serving", {"cell": cell_name, "serving_already": True}) @@ -1095,7 +1238,9 @@ async def attach_serving(cell_name: str, deployment_id: str, *, expected: Contro ) await _audit(conn, "attach_serving", {"cell": cell_name, "issuers": accepted}) written = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell = $1", cell_name) - return _attaching(written, None) + return _attaching(written, None) + + return await _attach_change(cell_name, expected, change_locked) def remove_refusal(cell_name: str, state: str, placed: int) -> str: @@ -1118,7 +1263,8 @@ async def remove(cell_name: str, *, expected: Control) -> Cell: first cell's Postgres reads `open` first, and a direct call skips that check, leaving a first cell still prepared answering its new orgs 503 and unable to register again until the full back-out has run.""" - async with _attach_change(cell_name, expected) as (conn, found): + + async def change_locked(conn: Any, found: Attaching) -> Cell: row = found.row if reason := remove_refusal(cell_name, row.state, row.placed): raise OperatorRefused(reason) @@ -1135,7 +1281,9 @@ async def remove(cell_name: str, *, expected: Control) -> Cell: "mode": row.mode, }, ) - return row + return row + + return await _attach_change(cell_name, expected, change_locked) async def lookup(org: str) -> tuple[str, str] | None: diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index fa375eb12a..e14eab1220 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -370,11 +370,13 @@ async def finalizer(request): await resume.wait() # request already sent; caller can no longer retract it async def change(): - async with directory._attach_change("first", compare) as (conn, found): + async def mutate(conn, found): await conn.execute( "UPDATE loom_router.cells SET current_fp = next_fp, next_fp = NULL, issuers = ARRAY['new'] WHERE cell = 'first'" ) - return found + return found + + return await directory._attach_change("first", compare, mutate) return await app._answered(change, app._attach_answer) diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py new file mode 100644 index 0000000000..9970d6b5ee --- /dev/null +++ b/tests/integration/test_directory_control_cleanup.py @@ -0,0 +1,296 @@ +"""Directory control closes its actual backend before an interrupted call returns.""" + +from __future__ import annotations + +import asyncio +import os +import time +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from server.router import directory +from server.settings import settings +from tests.integration._pg_database import create_database, drop_database +from tests.integration.test_pg_owned_cleanup import released, sleeping + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def directory_database(monkeypatch) -> AsyncIterator[Any]: + import asyncpg + + admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") + name, dsn = await create_database(admin) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + observer = await asyncpg.connect(dsn) + await observer.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") + try: + yield observer + finally: + await directory.close() + await observer.close(timeout=1) + await drop_database(admin, name) + + +async def locked(conn): + await conn.execute("SELECT pg_advisory_xact_lock($1)", directory._DIRECTORY_LOCK) + await conn.execute("SELECT * FROM probe FOR UPDATE") + + +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +async def test_directory_interrupt_releases_exact_backend_and_global_locks( + directory_database, monkeypatch, interrupt +): + seen = asyncio.Event() + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.5) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + task = asyncio.create_task(directory._control_transaction(work)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + began = time.monotonic() + if interrupt == "cancel": + task.cancel("original directory cancellation") + with pytest.raises( + asyncio.CancelledError if interrupt == "cancel" else directory.DirectoryUnavailable + ): + await asyncio.wait_for(task, 1) + assert time.monotonic() - began < 0.6 + await released(directory_database, backend["pid"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_original_failure_never_waits_for_rollback(directory_database, monkeypatch): + import asyncpg + + original = ValueError("original directory work failure") + real = asyncpg.Connection.execute + backend = {} + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) + + async def stalled(self, query, *args, **kwargs): + if query.startswith("ROLLBACK"): + backend["rollback"] = True + await real(self, "SELECT pg_sleep(10)") + return await real(self, query, *args, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "execute", stalled) + began = time.monotonic() + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + raise original + + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work) + assert caught.value is original and "rollback" not in backend + assert time.monotonic() - began < 0.4 + await released(directory_database, backend["pid"]) + + +async def test_directory_normal_commit_closes_before_return(directory_database): + backend = {} + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + return "acknowledged" + + assert await directory._control_transaction(work) == "acknowledged" + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_second_cancel_keeps_first_and_close_precedes_worker_cancel( + directory_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + seen, closing, allow_close = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + events = [] + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + try: + await conn.execute("SELECT pg_sleep(10)") + finally: + events.append("work stopped") + + async def close(self, **kwargs): + if self.get_server_pid() == backend.get("pid"): + events.append("close started") + await real_close(self, **kwargs) + closing.set() + await allow_close.wait() + else: + await real_close(self, **kwargs) + + monkeypatch.setattr(asyncpg.Connection, "close", close) + task = asyncio.create_task(directory._control_transaction(work)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory cancellation") + await asyncio.wait_for(closing.wait(), 1) + task.cancel("second directory cancellation") + with pytest.raises(asyncio.CancelledError, match="first directory cancellation"): + await asyncio.wait_for(task, 0.5) + assert events.index("close started") < events.index("work stopped") + assert events.count("close started") == 1 + await released(directory_database, backend["pid"]) + finally: + allow_close.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_directory_commit_acknowledgement_loss_is_not_success( + directory_database, monkeypatch +): + import asyncpg + + real = asyncpg.Connection.execute + backend = {} + original = OSError("commit acknowledgement lost") + + async def execute(self, sql, *args, **kwargs): + value = await real(self, sql, *args, **kwargs) + if sql.startswith("COMMIT") and self.get_server_pid() == backend.get("pid"): + raise original + return value + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "execute", execute) + with pytest.raises(directory.DirectoryUnavailable) as caught: + await directory._control_transaction(work) + assert caught.value.__cause__ is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +async def test_directory_required_close_failure_refuses_committed_success( + directory_database, monkeypatch +): + import asyncpg + + real_close = asyncpg.Connection.close + backend = {} + original = ValueError("required close failed") + + async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get("pid") + await real_close(self, **kwargs) + if ours: + raise original + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + await conn.execute("UPDATE probe SET id=2") + + monkeypatch.setattr(asyncpg.Connection, "close", close) + with pytest.raises(ValueError) as caught: + await directory._control_transaction(work) + assert caught.value is original + assert await directory_database.fetchval("SELECT id FROM probe") == 2 + await released(directory_database, backend["pid"]) + + +@pytest.mark.parametrize("stage", ["acquire", "work", "commit", "close", "release"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_directory_fatal_child_is_inert_until_owned_cleanup(directory_database, stage, kind): + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await directory_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, contextlib, os, sys +import asyncpg +from server.router import directory +from server.settings import settings +stage, kind = sys.argv[1:] +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('fatal guard') +async def main(): + settings.router_pg_dsn = os.environ['CONTROL_TEST_DSN'] + observer = await asyncpg.connect(settings.router_pg_dsn) + backend = {} + real_close = asyncpg.Connection.close + real_execute = asyncpg.Connection.execute + real_connection = directory._connection + async def execute(self, sql, *args, **kw): + if stage == 'commit' and sql.startswith('COMMIT'): raise original + return await real_execute(self, sql, *args, **kw) + async def close(self, **kw): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kw) + if ours and stage == 'close': raise original + @contextlib.asynccontextmanager + async def connection(): + if stage == 'acquire': raise original + async with real_connection() as conn: + yield conn + if stage == 'release': raise original + async def work(conn): + backend['pid'] = conn.get_server_pid() + await conn.execute('SELECT pg_advisory_xact_lock($1)', directory._DIRECTORY_LOCK) + await conn.execute('SELECT * FROM probe FOR UPDATE') + if stage == 'work': raise original + asyncpg.Connection.close = close + asyncpg.Connection.execute = execute + directory._connection = connection + try: + try: await directory._control_transaction(work) + except (SystemExit, KeyboardInterrupt) as caught: assert caught is original + else: raise AssertionError('fatal guard returned success') + if stage == 'acquire': + assert not backend and directory._pool is None + else: + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + finally: + asyncpg.Connection.close = real_close + asyncpg.Connection.execute = real_execute + await directory.close() + await observer.close(timeout=1) +asyncio.run(main()) +""" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + stage, + kind, + env={**os.environ, "CONTROL_TEST_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), 10) + assert process.returncode == 0, (stdout.decode(), stderr.decode()) diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index cbb96ba4cc..41f8d343c6 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -7,7 +7,6 @@ from __future__ import annotations import asyncio -import contextlib import hashlib import json import os @@ -828,12 +827,14 @@ async def test_a_live_placement_racing_a_batch_of_the_same_org_answers_it_placed written, release = asyncio.Event(), asyncio.Event() real = directory._attach_change - @contextlib.asynccontextmanager - async def _held(cell_name: str, expected: directory.Control) -> Any: - async with real(cell_name, expected) as locked: - yield locked + async def _held(cell_name: str, expected: directory.Control, change) -> Any: + async def hold(conn, found): + result = await change(conn, found) written.set() await release.wait() # the batch's writes are done, its commit held + return result + + return await real(cell_name, expected, hold) monkeypatch.setattr(directory, "_attach_change", _held) conn = await asyncpg.connect(own_dsn) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 20e17ac786..11bf1bf094 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -7,9 +7,10 @@ import pytest -from server.cli import cell_attach -from server.cli import cell_attach_pg as apg +from server.cli import cell_attach, cell_control from server.core import pg +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_control import snapshot @pytest.mark.asyncio @@ -24,12 +25,16 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + original = ValueError("original work failure") + async def work(): - raise ValueError("original work failure") + raise original - with pytest.raises(ValueError, match="original work failure"): - await apg.supervise(Conn(), work(), end=time.monotonic() + 0.1) - assert events == ["close", "abort"] + with pytest.raises(ValueError, match="original work failure") as caught: + await owner(Conn()).call(work, total_s=0.1) + assert caught.value is original + assert events[0] == "close" and events.count("close") == 1 + assert events[1:] and set(events[1:]) == {"abort"} @pytest.mark.asyncio @@ -56,7 +61,7 @@ async def work(): finally: events.append("work ended") - task = asyncio.create_task(apg.supervise(Conn(), work(), end=time.monotonic() + 0.2)) + task = asyncio.create_task(owner(Conn()).call(work, total_s=0.2)) await entered.wait() if not failed: task.cancel("first cancellation") @@ -73,47 +78,58 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_context_exit_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): + closing = asyncio.Event() conn = AsyncMock() - conn.is_closed = lambda: False conn.terminate = lambda: None - cm = AsyncMock() - exit_started = asyncio.Event() - async def exit(*_): - exit_started.set() + async def close(**_): + closing.set() await asyncio.Future() - cm.__aexit__.side_effect = exit - run = cell_attach.Run(None, argparse.Namespace(cell="test", wait=0), None, lambda _: None) - run._lock_cm, run.lock_conn = cm, conn + conn.close.side_effect = close + owned = owner(conn) monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) started = time.monotonic() with pytest.raises(TimeoutError): - await run.close_lock() + await owned.close() assert time.monotonic() - started < 0.15 - assert exit_started.is_set() and run.lock_conn is None + assert closing.is_set() and owned.closed conn.close.assert_awaited_once() await asyncio.sleep(0) @pytest.mark.asyncio @pytest.mark.parametrize("failed", [False, True]) -async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure(failed): +async def test_run_preserves_its_own_failure_without_hiding_normal_close_failure( + failed, monkeypatch +): router = AsyncMock() - router.call.return_value = {"state": "attaching"} + router.call.return_value = {**snapshot(), "state": "attaching"} run = cell_attach.Run( None, argparse.Namespace(cell="test", wait=0, dry_run=True), router, lambda _: None ) - run.open_lock = AsyncMock(side_effect=ValueError("run failed") if failed else None) + original = ValueError("run failed") + conn = AsyncMock() + conn.terminate = lambda: None + conn.close.side_effect = RuntimeError("close failed") + + async def open(owned, what): + owned.conn = conn + if failed: + raise original + + monkeypatch.setattr(cell_control.Owner, "_open", open) run.read_own_row = AsyncMock() run.refusals = AsyncMock(return_value=(None, None, None)) run.dry_run = AsyncMock(return_value=0) - run.close_lock = AsyncMock(side_effect=RuntimeError("close failed")) try: raise LookupError("unrelated caller exception") except LookupError: with pytest.raises( ValueError if failed else RuntimeError, match="run failed" if failed else "close failed" - ): + ) as caught: await run.run() + if failed: + assert caught.value is original + conn.close.assert_awaited_once() diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 0c32387393..0c358f2c38 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -116,15 +116,15 @@ def test_cell_control_migrations_are_classified(): "server/router/directory.py": { ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, - ("claim", "UPDATE", "loom_router.cells"): 1, - ("attach_begin", "UPDATE", "loom_router.cells"): 1, - ("attach_place", "INSERT INTO", "loom_router.placements"): 1, - ("attach_place", "UPDATE", "loom_router.cells"): 1, - ("attach_place", "INSERT INTO", "loom_router.audit"): 1, - ("attach_enforced", "UPDATE", "loom_router.cells"): 1, - ("attach_serving", "UPDATE", "loom_router.cells"): 1, - ("remove", "DELETE FROM", "loom_router.hosts"): 1, - ("remove", "DELETE FROM", "loom_router.cells"): 1, + ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_begin.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.placements"): 1, + ("attach_place.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_place.change_locked", "INSERT INTO", "loom_router.audit"): 1, + ("attach_enforced.change_locked", "UPDATE", "loom_router.cells"): 1, + ("attach_serving.change_locked", "UPDATE", "loom_router.cells"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.hosts"): 1, + ("remove.change_locked", "DELETE FROM", "loom_router.cells"): 1, }, } @@ -387,15 +387,39 @@ def function(path: str, name: str) -> ast.AST: "cell_control.check_lock(conn)", ], ), + ( + "server/router/directory.py", + "_control_transaction", + [ + "owner.start(lambda: owner.work(change))", + "owner.wait(task, owner.work_end)", + "owner.finish(task)", + ], + ), + ( + "server/router/directory.py", + "_ControlOwner.work", + [ + "self.context.__aenter__()", + "self.conn.transaction()", + "tx.start()", + "change(self.conn)", + "tx.commit()", + ], + ), ( "server/router/directory.py", "_attach_change", - ["_control_transaction()", "control_refusal(attaching, expected, claim=claim)"], + [ + "_control_transaction(locked)", + "control_refusal(attaching, expected, claim=claim)", + "change(conn, attaching)", + ], ), ( "server/router/directory.py", "_operator_change", - ["_attach_change(cell_name, expected)", "change(conn, found.row)"], + ["_attach_change(cell_name, expected, change_locked)", "change(conn, found.row)"], ), ( "server/router/directory.py", @@ -410,10 +434,14 @@ def function(path: str, name: str) -> ast.AST: ( "server/router/directory.py", "claim", - ["_attach_change(cell_name, expected, claim=True)"], + ["_attach_change(cell_name, expected, change_locked, claim=True)"], ), *[ - ("server/router/directory.py", name, ["_attach_change(cell_name, expected)"]) + ( + "server/router/directory.py", + name, + ["_attach_change(cell_name, expected, change_locked)"], + ) for name in ( "attach_begin", "attach_place", diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py new file mode 100644 index 0000000000..1a66b20c26 --- /dev/null +++ b/tests/unit/test_directory_control_cleanup.py @@ -0,0 +1,136 @@ +"""A directory control attempt owns acquisition, cleanup and context exit once.""" + +import asyncio +import contextlib +import time +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def owned(monkeypatch): + conn = Mock() + conn.transaction.return_value = Mock(start=AsyncMock(), commit=AsyncMock()) + conn.close = AsyncMock() + conn.terminate = Mock() + context = Mock(__aenter__=AsyncMock(return_value=conn), __aexit__=AsyncMock(return_value=None)) + monkeypatch.setattr(directory, "_connection", lambda: context) + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.08) + return conn, context + + +async def test_success_closes_once_before_pool_exit(owned): + conn, context = owned + events = [] + conn.close.side_effect = lambda **_: events.append("close") + context.__aexit__.side_effect = lambda *_: events.append("exit") + assert await directory._control_transaction(AsyncMock(return_value=19)) == 19 + assert events == ["close", "exit"] + conn.close.assert_awaited_once() + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_original_error_survives_close_and_exit_failure(owned): + conn, context = owned + original = ValueError("work failure") + conn.close.side_effect = RuntimeError("close failure") + context.__aexit__.side_effect = LookupError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock(side_effect=original)) + assert caught.value is original + conn.close.assert_awaited_once() + context.__aexit__.assert_awaited_once() + conn.transaction.return_value.commit.assert_not_awaited() + + +async def test_close_failure_survives_exit_failure_after_acknowledged_commit(owned): + conn, context = owned + original = ValueError("close failure") + conn.close.side_effect = original + context.__aexit__.side_effect = RuntimeError("exit failure") + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock()) + assert caught.value is original + conn.transaction.return_value.commit.assert_awaited_once() + + +async def test_exit_cannot_open_an_extra_cleanup_allowance(owned): + conn, context = owned + entered = asyncio.Event() + + async def stalled(*_): + entered.set() + await asyncio.Future() + + context.__aexit__.side_effect = stalled + began = time.monotonic() + with pytest.raises(directory.DirectoryUnavailable): + await directory._control_transaction(AsyncMock()) + assert entered.is_set() and time.monotonic() - began < 0.2 + conn.close.assert_awaited_once() + + +async def test_acquisition_failure_does_not_close_an_unowned_proxy(owned): + conn, context = owned + original = ValueError("acquisition failure") + context.__aenter__.side_effect = original + with pytest.raises(ValueError) as caught: + await directory._control_transaction(AsyncMock()) + assert caught.value is original + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + + +async def test_no_known_proxy_cancellation_does_not_close_another_attempt(owned): + conn, context = owned + entered = asyncio.Event() + + async def acquire(): + entered.set() + await asyncio.Future() + + context.__aenter__.side_effect = acquire + task = asyncio.create_task(directory._control_transaction(AsyncMock())) + await entered.wait() + task.cancel("first acquisition cancellation") + with pytest.raises(asyncio.CancelledError, match="first acquisition cancellation"): + await task + conn.close.assert_not_awaited() + conn.terminate.assert_not_called() + context.__aexit__.assert_not_awaited() + context.__aenter__.side_effect = None + assert await directory._control_transaction(AsyncMock(return_value=2)) == 2 + conn.close.assert_awaited_once() + + +async def test_late_acquired_proxy_is_discarded_without_running_callback(owned): + conn, context = owned + entered, allow = asyncio.Event(), asyncio.Event() + + async def acquire(): + entered.set() + while not allow.is_set(): + with contextlib.suppress(asyncio.CancelledError): + await allow.wait() + return conn + + context.__aenter__.side_effect = acquire + work = AsyncMock() + task = asyncio.create_task(directory._control_transaction(work)) + await entered.wait() + task.cancel("acquisition cancelled") + with pytest.raises(asyncio.CancelledError, match="acquisition cancelled"): + await task + allow.set() + for _ in range(20): + await asyncio.sleep(0) + work.assert_not_awaited() + conn.transaction.assert_not_called() + conn.terminate.assert_called() + conn.close.assert_not_awaited() # Expired original end cannot create a fresh close budget. diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 226a1b54c0..bffc8aefea 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -89,6 +89,7 @@ class _Conn: def __init__(self, store: _Store) -> None: self.s = store + self._tx = None def _count(self) -> None: if self.s.down: @@ -175,8 +176,13 @@ async def fetch(self, sql: str, *a: Any) -> list[Any]: self.s.placements.update(dict.fromkeys(new, a[1])) return [{"org": o} for o in new] + async def close(self, **_kw: Any) -> None: + self.terminate() + def terminate(self) -> None: - pass + if self._tx is not None: + self.s.cells, self.s.hosts, self.s.placements, self.s.audit = self._tx.saved + self._tx = None def transaction(self) -> Any: conn = self @@ -192,12 +198,14 @@ async def rollback(self) -> None: await self.__aexit__(Exception) async def __aenter__(self) -> None: + conn._tx = self conn._count() self.saved = copy.deepcopy( (conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit) ) async def __aexit__(self, kind: Any, *_exc: Any) -> None: + conn._tx = None conn.s.statements += 1 # the commit, or the rollback if kind is not None: conn.s.cells, conn.s.hosts, conn.s.placements, conn.s.audit = self.saved From 0c106360020e8869752e188d60ae5ab2344b4618 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:19:39 +0000 Subject: [PATCH 057/181] Measure cell triggers with the original API and delivery workload Use five runs per arm with 2,000 sequential 4-KiB values and five real worker batches at the configured size. Verify the owned fixture and stored results, retain raw samples, and restore triggers before a final verdict. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 1306 ++++++++++++++++++-- tests/unit/test_bench_cell_trigger_gate.py | 990 ++++++++++++++- 2 files changed, 2162 insertions(+), 134 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 8797b4b8e7..5bd3bfd3db 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1,134 +1,1244 @@ -"""The first attach's trigger-cost gate, end to end: `memory.set` p50 and p99 through a server -of an attached cell, and a delivery-sized 1,000-row insert of a held org from a marked and an -unmarked session, with the cell's org triggers on and dropped in turn (the `loom.tenancy` -guard stays), deleting the rows its inserts wrote at the end. Run it against an attached -cell's server, which must not serve anything else, with a writer token for a placed org; -it drops and recreates the cell's org triggers, so never against a deployment in use. -Needs no provider. - - PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \\ - --base-url http://127.0.0.1:7932 --pg-dsn \\ - --token --org --namespace +"""Measure API p99 and real delivery throughput with cell org triggers on and off. + +Requires an independently owned, attached scratch fixture and its source/ownership +manifest. The outer fixture owner stops its processes and drops its exact databases. +This driver restores triggers but never deletes fixture rows or changes providers. """ from __future__ import annotations import argparse import asyncio +import hashlib +import ipaddress +import json +import math +import os +import re import statistics +import subprocess +import sys import time import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, TextIO, cast +from urllib.parse import unquote, urlsplit import asyncpg +import clickhouse_connect import httpx from server.core import cell_triggers +from server.core.hashing import hash_text, sha256_value + +CALLS, WARMUP, ROUNDS, BATCHES = 2000, 50, 5, 5 +VALUE_BYTES, KEY_CEILING = 4096, 100_000 +IO_S, SETTLE_S, RESTORE_S = 30.0, 60.0, 15.0 +SUBJECT = "process:benchmark" +_SHA = re.compile(r"[a-f0-9]{64}") +_DB = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*") +_DEFAULT_DBS = { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", +} + + +class Invalid(ValueError): + """The fixture or a sample did not meet the declared measurement contract.""" + + +def _require(ok: object, field: str) -> None: + if not ok: + raise Invalid(field) + + +def _planned_keys(batch: int) -> int: + _require(type(batch) is int and batch > 0, "delivery batch") + count = 2 * ROUNDS * (CALLS + WARMUP + BATCHES * batch) + _require(count <= KEY_CEILING, "100000 key ceiling") + return count + + +def _arm_order() -> tuple[bool, ...]: + return tuple( + on for pair in range(ROUNDS) for on in ((True, False) if pair % 2 == 0 else (False, True)) + ) + -CALLS, WARMUP, ROUNDS, INSERTS = 400, 50, 3, 20 +def _payload(key: str) -> str: + prefix = f"Cell trigger measurement {key}: " + _require(len(prefix) < VALUE_BYTES and prefix.isascii(), "payload key") + return prefix + ("abcdefghijklmnopqrstuvwxyz " * VALUE_BYTES)[: VALUE_BYTES - len(prefix)] -async def _triggers(dsn: str, on: bool) -> None: - """Create or drop the org triggers, and check that what was asked holds: every guarded - table carries both triggers enabled at origin, or none does.""" - conn = await asyncpg.connect(dsn) +def _endpoint(value: str, *, pg: bool = False) -> tuple[str, str, int, str]: + """Validate the literal target without resolving names or accepting DSN overrides.""" try: - guarded = [t for t in await cell_triggers.planned_tables(conn) if t != "tenancy"] - for table in guarded: - for name in cell_triggers.ORG_TRIGGERS: - await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') - if on: - await conn.execute(cell_triggers.create_statement(table, name)) - enabled = await conn.fetchval( - "SELECT count(*) FROM pg_trigger WHERE tgenabled = 'O' AND tgname = ANY($1::text[])", - list(cell_triggers.ORG_TRIGGERS), + u = urlsplit(value) + host, port = u.hostname, u.port + _require( + u.scheme in ({"postgres", "postgresql"} if pg else {"http", "https"}), "endpoint scheme" + ) + _require( + host and ipaddress.ip_address(host).is_loopback and port and 0 < port < 65536, + "numeric loopback endpoint", ) + _require(not u.query and not u.fragment, "endpoint overrides") + if not pg: + _require(not u.username and not u.password and u.path in ("", "/"), "HTTP endpoint") + database = unquote(u.path.removeprefix("/")) if pg else "" + if pg: + _scratch(database) + return u.scheme, str(ipaddress.ip_address(str(host))), int(port or 0), database + except (ValueError, TypeError): + raise Invalid("invalid local endpoint") from None + + +def _scratch(value: str) -> None: + _require( + bool(_DB.fullmatch(value)) and value.lower() not in _DEFAULT_DBS, + "explicit scratch database", + ) + + +def _digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +@dataclass(frozen=True) +class Fixture: + """Nonsecret receipt from the owner that created and launched this exact fixture.""" + + base_url: str + pg_host: str + pg_port: int + pg_database: str + ch_host: str + ch_port: int + ch_database: str + ch_telemetry_database: str + cell: str + org: str + namespace: str + deployment_id: str + epoch: int + instance_id: str + build_commit: str + source_tree: str + launch_manifest_sha256: str + ownership_receipt_sha256: str + configuration_sha256: str + background_workers: str + controller_excluded: bool + pg_internal_port: int | None = None + + @classmethod + def read(cls, path: str) -> Fixture: + try: + with Path(path).open("rb") as manifest_file: + raw = manifest_file.read(16_385) + _require(len(raw) <= 16_384, "manifest size") + return cls(**json.loads(raw)) + except (TypeError, ValueError, OSError): + raise Invalid("fixture manifest") from None + + def validate(self, args: argparse.Namespace, config: Any) -> int: + supplied = _endpoint(args.pg_dsn, pg=True) + configured = _endpoint(config.pg_dsn, pg=True) + _require(supplied == configured, "configured PG endpoint") + _require( + supplied[1:] == (self.pg_host, self.pg_port, self.pg_database), "manifest PG endpoint" + ) + _require(_endpoint(args.base_url) == _endpoint(self.base_url), "manifest HTTP endpoint") + for host, port in ((self.ch_host, self.ch_port), (config.ch_host, config.ch_port)): + _endpoint(f"http://[{host}]:{port}" if ":" in host else f"http://{host}:{port}") + _require( + (self.ch_host, self.ch_port) == (config.ch_host, config.ch_port), "manifest CH endpoint" + ) + for name in (self.pg_database, self.ch_database, self.ch_telemetry_database): + _scratch(name) + _require( + (self.ch_database, self.ch_telemetry_database) + == (config.ch_database, config.ch_telemetry_database), + "manifest CH aliases", + ) + _require( + (self.org, self.namespace, self.cell) == (args.org, args.namespace, config.cell), + "manifest identity", + ) + _require( + all((self.org, self.namespace, self.cell, self.instance_id, self.deployment_id)), + "manifest identity fields", + ) + _require( + self.background_workers == config.background_workers == "off" + and self.controller_excluded is True, + "exclusive fixture", + ) + _require( + config.storage_authority == "postgres" + and config.embedding_provider == "fake" + and config.llm_provider == "fake", + "explicit fake providers", + ) + _require( + self.build_commit == config.build_commit + and bool(re.fullmatch(r"[a-f0-9]{40}", self.build_commit)), + "source build", + ) + _require(bool(re.fullmatch(r"[a-f0-9]{40}", self.source_tree)), "source tree") + for value in ( + self.launch_manifest_sha256, + self.ownership_receipt_sha256, + self.configuration_sha256, + ): + _require(bool(_SHA.fullmatch(value)), "fixture receipt digest") + _require( + self.configuration_sha256 == _digest(_configuration(config)), "fixture configuration" + ) + _require(type(self.epoch) is int and self.epoch > 0, "authority epoch") + _require( + self.pg_internal_port is None + or type(self.pg_internal_port) is int + and 0 < self.pg_internal_port < 65536, + "internal PG port", + ) + return _planned_keys(config.pg_delivery_batch_rows) + + +def _configuration(config: Any) -> dict[str, Any]: + # A complete digest catches a setting change without publishing credentials. + return cast(dict[str, Any], config.model_dump(mode="json")) + + +def _source(fixture: Fixture) -> dict[str, Any]: + """Fixed read-only Git commands bind the launched build to this clean checkout.""" + root = Path(__file__).resolve().parents[1] + values = [] + for command in (("rev-parse", "HEAD"), ("rev-parse", "HEAD^{tree}"), ("status", "--porcelain")): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, timeout=5, check=False + ) + _require(result.returncode == 0, "source Git read") + _require(len(result.stdout) <= 65_536, "source status size") + values.append(result.stdout.strip()) + head, tree, status = values + _require( + head == fixture.build_commit and tree == fixture.source_tree and not status, + "clean source head/tree", + ) + return {"head": head, "tree": tree, "clean": True} + + +class Artifact: + def __init__(self, path: str): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + self.file: TextIO = os.fdopen(fd, "w", encoding="utf-8") + self.records = 0 + + def emit(self, kind: str, **record: Any) -> None: + self.records += 1 + _require(self.records <= KEY_CEILING + 1000, "output record ceiling") + line = json.dumps({"kind": kind, **record}, sort_keys=True, allow_nan=False) + _require(len(line) <= 65_536, "output record size") + self.file.write(line + "\n") + + def close(self) -> None: + try: + self.file.flush() + os.fsync(self.file.fileno()) + finally: + self.file.close() + + +async def _close_pg(conn: Any) -> None: + from server.core import pg + + await pg.close_owned(conn, end=asyncio.get_running_loop().time() + IO_S) + + +class ReadOnlyCH: + """Public ClickHouse client restricted at connection startup, before any query.""" + + def __init__(self, client: Any): + self.client = client + + @classmethod + async def open(cls) -> ReadOnlyCH: + from server.core.ch_names import NamedClient + from server.core.ch_target import connection_kwargs + + raw = await clickhouse_connect.get_async_client( + **connection_kwargs(), + settings={ + "readonly": 1, + "max_execution_time": int(IO_S), + "log_comment": "bench.cell_trigger_gate.preflight", + }, + connect_timeout=IO_S, + send_receive_timeout=IO_S, + ) + return cls(NamedClient(raw)) + + async def query_all( + self, sql: str, params: dict[str, Any] | None = None + ) -> list[dict[str, Any]]: + result = await self.client.query(sql, parameters=params or {}) + return list(result.named_results()) + + async def close(self) -> None: + async with asyncio.timeout(IO_S): + await self.client.close() + + +async def _health(args: argparse.Namespace, fixture: Fixture) -> None: + async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: + health = await client.get(f"{args.base_url}/v1/health") + deployment = await client.get(f"{args.base_url}/v1/health/deployment") + _require(health.status_code == deployment.status_code == 200, "health HTTP status") + h, d = health.json(), deployment.json() + _require( + h.get("build_commit") == fixture.build_commit + and d.get("deployment_id") == fixture.deployment_id, + "serving source/deployment", + ) + _require( + h.get("deployment_config", {}).get("instance_id") == fixture.instance_id, + "serving instance", + ) + _require( + h.get("ch_ok") is True and h.get("pg_ok") is True and h.get("status") == "ok", + "serving health", + ) + _require( + h.get("fake_embedding") is True + and h.get("fake_llm") is True + and h.get("pending_migrations") == [], + "serving provider/schema", + ) + _require(h.get("storage_authority") == "postgres", "serving authority") + + +def _object(value: Any) -> dict[str, Any]: + if isinstance(value, str): + value = json.loads(value) + _require(isinstance(value, dict), "metadata JSON object") + return cast(dict[str, Any], value) + + +async def _proof( + conn: Any, ch: Any, args: argparse.Namespace, fixture: Fixture, *, native: bool = False +) -> dict[str, Any]: + from server.core import cell, instances + from server.settings import settings + + fixture.validate(args, settings) + source = _source(fixture) + actual = await conn.fetchrow( + "SELECT current_database() AS db, inet_server_port() AS port, " + "pg_is_in_recovery() AS recovery" + ) + _require( + actual and actual["db"] == fixture.pg_database and actual["recovery"] is False, + "actual PG identity/primary", + ) + if fixture.pg_internal_port is not None: + _require(actual["port"] == fixture.pg_internal_port, "PG internal port") + authority = await conn.fetchrow("SELECT authority, deployment_id, epoch FROM loom.authority") + _require( + authority + and (authority["authority"], authority["deployment_id"], authority["epoch"]) + == ("postgres", fixture.deployment_id, fixture.epoch), + "PG authority identity", + ) + first = await conn.fetchrow("SELECT is_first, cell FROM loom.cell_first") + mode = await conn.fetchval("SELECT mode FROM loom.tenancy_mode") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + held = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM loom.tenancy WHERE org = $1)", args.org + ) + _require( + first and first["cell"] == fixture.cell and mode == "enforced" and held is True, + "enforced held org", + ) + _require(own and own["next_fp"] is None, "stable accepted fingerprint") + current = _object(own["current_fp"]) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + _require(cell.matches(mine, current), "accepted local fingerprint") + db = await ch.query_all("SELECT currentDatabase() AS db") + _require(db == [{"db": fixture.ch_database}], "actual CH database") + records = await ch.query_all( + "SELECT authority, epoch, deployment_id FROM loom.storage_authority FINAL" + ) + _require(len(records) == 1 and records[0] == dict(authority), "paired CH authority") + for table in ("provider_configuration", "runtime_overrides"): + rows = await ch.query_all(f"SELECT count() AS n FROM loom.{table}") + _require(rows == [{"n": 0}], f"fresh {table}") + live = await instances.read_live(ch, all_deployments=True) + _require( + len(live) == 1 + and live[0]["instance_id"] == fixture.instance_id + and live[0]["build_commit"] == fixture.build_commit + and not live[0]["workers_lease"], + "exclusive serving source", + ) + beats = await ch.query_all( + "SELECT instance_id, cell, tenancy_mode, fingerprint FROM " + "loom.cell_heartbeats WHERE instance_id = {id:String}", + {"id": fixture.instance_id}, + ) + _require( + len(beats) == 1 + and beats[0]["cell"] == fixture.cell + and beats[0]["tenancy_mode"] == "enforced", + "serving cell heartbeat", + ) + reported = _object(beats[0]["fingerprint"]) + _require( + reported + and cell.matches(mine, reported) + and reported.get("serves") is True + and reported.get("marked") is True, + "serving fingerprint", + ) + await _debt(conn, 0) + await _health(args, fixture) + if native: + await cell.refresh() + _require(not cell.cli_gate_reason(), "native fresh cell acceptance") + return { + "deployment_id": fixture.deployment_id, + "epoch": fixture.epoch, + "instance_id": fixture.instance_id, + "fingerprint_sha256": _digest(current), + "configuration_sha256": fixture.configuration_sha256, + "source": source, + } + + +async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: + conn = ch = None + try: + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"default_transaction_read_only": "on"}, + ) + ch = await ReadOnlyCH.open() + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture) finally: - await conn.close() - expected = 2 * len(guarded) if on else 0 - if enabled != expected: - raise SystemExit(f"{enabled} org triggers enabled, expected {expected}") + try: + if ch is not None: + await ch.close() + finally: + if conn is not None: + await _close_pg(conn) + + +async def _debt(conn: Any, expected: int) -> None: + row = await conn.fetchrow( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE targets <> " + "0) AS pending, (SELECT count(*) FROM loom.audit_outbox) AS audits, " + "(SELECT count(*) FROM loom.memory_deliveries WHERE attempts <> 0 OR " + "last_error <> '') AS errors" + ) + _require( + row and row["pending"] == expected and row["audits"] == row["errors"] == 0, + "fixture delivery/audit debt", + ) + + +@dataclass +class Triggers: + tables: tuple[str, ...] + invariant: str + org: str + on: bool = True + + @staticmethod + async def snapshot(conn: Any) -> tuple[tuple[str, ...], str, str]: + tables = tuple(await cell_triggers.planned_tables(conn)) + rows = await conn.fetch( + "SELECT c.relname AS table_name, t.tgname AS name, t.tgenabled AS " + "enabled, pg_get_triggerdef(t.oid) AS definition FROM pg_trigger t " + "JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid " + "= c.relnamespace WHERE n.nspname = 'loom' AND NOT t.tgisinternal " + "ORDER BY c.relname, t.tgname" + ) + functions = await conn.fetch( + "SELECT p.proname, pg_get_functiondef(p.oid) AS definition FROM " + "pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE " + "n.nspname = 'loom' AND p.prokind = 'f' ORDER BY p.proname, p.oid" + ) + org = [dict(r) for r in rows if r["name"] in cell_triggers.ORG_TRIGGERS] + other = [dict(r) for r in rows if r["name"] not in cell_triggers.ORG_TRIGGERS] + return ( + tables, + _digest({"functions": [dict(r) for r in functions], "triggers": other}), + _digest(org), + ) + + @classmethod + async def capture(cls, conn: Any) -> Triggers: + _require(not await cell_triggers.missing(conn), "canonical ON trigger baseline") + tables, invariant, org = await cls.snapshot(conn) + _require(tables, "guarded tables") + return cls(tables, invariant, org) + + async def verify(self, conn: Any) -> None: + tables, invariant, org = await self.snapshot(conn) + _require( + tables == self.tables and invariant == self.invariant, + "trigger table/function/tenancy drift", + ) + _require(org == (self.org if self.on else _digest([])), "prior trigger treatment") + + async def set(self, conn: Any, on: bool) -> None: + await self.verify(conn) + async with conn.transaction(): + for table in self.tables: + if table == "tenancy": + continue + _require(bool(_DB.fullmatch(table)), "guarded table identifier") + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + if on: + await conn.execute(cell_triggers.create_statement(table, name)) + self.on = on + await self.verify(conn) + + async def restore(self, conn: Any) -> None: + # Recreate the captured exact org guards even after a partial failed toggle. + async with conn.transaction(): + for table in self.tables: + if table == "tenancy": + continue + for name in cell_triggers.ORG_TRIGGERS: + await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') + await conn.execute(cell_triggers.create_statement(table, name)) + self.on = True + await self.verify(conn) + _require(not await cell_triggers.missing(conn), "restored canonical triggers") -async def _memory_set(args: argparse.Namespace) -> tuple[float, float]: - """p50 and p99 in ms of CALLS sequential `memory.set` calls, each a new key.""" - lat = [] - headers = {"Authorization": f"Bearer {args.token}"} - async with httpx.AsyncClient(timeout=30, headers=headers) as client: +@dataclass +class Samples: + on: bool + api_ms: list[float] + delivery_rps: float + + +def _quantile(values: Sequence[float], fraction: float) -> float: + _require(values and all(math.isfinite(v) and v >= 0 for v in values), "finite latency samples") + return sorted(values)[math.ceil(fraction * len(values)) - 1] + + +def _statistics(samples: Sequence[Samples]) -> dict[str, Any]: + _require(len(samples) == 2 * ROUNDS, "complete arm samples") + arms: dict[bool, dict[str, Any]] = {} + for on in (True, False): + arm = [s for s in samples if s.on is on] + _require( + len(arm) == ROUNDS and all(len(s.api_ms) == CALLS for s in arm), "complete run samples" + ) + p99 = [_quantile(s.api_ms, 0.99) for s in arm] + rps = [s.delivery_rps for s in arm] + _require(all(math.isfinite(r) and r > 0 for r in rps), "positive finite throughput") + arms[on] = { + "p99_ms": p99, + "median_p99_ms": statistics.median(p99), + "min_p99_ms": min(p99), + "max_p99_ms": max(p99), + "rps": rps, + "median_rps": statistics.median(rps), + } + delta = arms[True]["median_p99_ms"] - arms[False]["median_p99_ms"] + loss = 1 - arms[True]["median_rps"] / arms[False]["median_rps"] + paired = [] + for i in range(0, len(samples), 2): + a, b = samples[i : i + 2] + _require(a.on is not b.on, "paired arm order") + paired.append((_quantile(a.api_ms, 0.99) - _quantile(b.api_ms, 0.99)) * (1 if a.on else -1)) + return { + "on": arms[True], + "off": arms[False], + "p99_delta_ms": delta, + "throughput_loss": loss, + "latency_within_limit": delta <= 10, + "throughput_within_limit": loss <= 0.10, + "paired_p99_differences_ms": paired, + } + + +@dataclass +class RequestRow: + key: str + memory_id: str + version: int + content_sha256: str + elapsed_ms: float + measured: bool + admission: str + + +async def _memory_set( + args: argparse.Namespace, *, arm: int = 0, emit: Callable[..., None] = lambda *_a, **_kw: None +) -> list[RequestRow]: + rows = [] + prefix = f"gate-{uuid.uuid4().hex}-{arm}" + async with httpx.AsyncClient( + timeout=IO_S, + headers={"Authorization": f"Bearer {args.token}"}, + follow_redirects=False, + trust_env=False, + ) as client: for i in range(CALLS + WARMUP): - body = { - "org": args.org, - "namespace": args.namespace, - "agent": "bench", - "memory_key": f"gate-{uuid.uuid4().hex[:10]}", - "value": f"trigger gate {i}", - } - t = time.perf_counter() - (await client.post(f"{args.base_url}/v1/memory.set", json=body)).raise_for_status() - if i >= WARMUP: - lat.append((time.perf_counter() - t) * 1000) - lat.sort() - return lat[len(lat) // 2], lat[int(0.99 * (len(lat) - 1))] - - -async def _batch(args: argparse.Namespace, marked: bool) -> float: - """The median in ms of INSERTS 1,000-row inserts of the held org, in one statement each.""" - settings = {"loom.tenancy_checked": "on"} if marked else {} - conn = await asyncpg.connect(args.pg_dsn, server_settings=settings) + key = f"{prefix}-{i}" + value = _payload(key) + started = time.perf_counter_ns() + try: + response = await client.post( + f"{args.base_url}/v1/memory.set", + json={ + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": value, + }, + ) + _require(response.status_code == 200, "memory.set HTTP status") + out = response.json() + _require( + isinstance(out, dict) + and out.get("status") == "active" + and not out.get("noop") + and not out.get("degraded") + and not out.get("duplicate_of") + and not out.get("merged_into_key"), + "memory.set outcome", + ) + memory_id = str(uuid.UUID(out["memory_id"])) + _require( + type(out.get("version")) is int and out["version"] > 0, "memory.set version" + ) + _require( + out.get("memory_key") == key + and out.get("namespace") == args.namespace + and out.get("content_sha256") == sha256_value(value), + "memory.set identity/hash", + ) + admission = "derived_ready" + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + _require(math.isfinite(elapsed) and elapsed >= 0, "memory.set elapsed") + row = RequestRow( + key, + memory_id, + out["version"], + sha256_value(value), + elapsed, + i >= WARMUP, + admission, + ) + rows.append(row) + emit("request", arm=arm, index=i, status=200, **asdict(row)) + except BaseException as error: + emit("request_error", arm=arm, index=i, key=key, error_type=type(error).__name__) + raise + return rows + + +@dataclass +class Block: + rows: list[dict[str, Any]] + audits: list[dict[str, Any]] + documents: list[dict[str, Any]] + seqs: list[tuple[str, str, int]] = field(default_factory=list) + + +async def _build_block(args: argparse.Namespace, batch: int) -> Block: + from server.llm import customer + from server.memory.store import pg_memory, pg_publish + from server.memory.store.rows import ( + MemoryRow, + build_memory_version_row, + next_version, + ) + + _planned_keys(batch) + prefix = f"gate-delivery-{uuid.uuid4().hex}" + rows, audits, documents = [], [], [] + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + for index in range(BATCHES * batch): + key = f"{prefix}-{index}" + value = _payload(key) + stored = json.dumps(value, sort_keys=True, separators=(",", ":")) + row = build_memory_version_row( + MemoryRow.from_snapshot( + { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": stored, + "version": next_version(), + "content_sha256": sha256_value(value), + "actor": SUBJECT, + "source_class": "asserted", + } + ) + ) + row = pg_memory.decode_row(pg_memory.encode_row(row)) + document = await _expected_document(args, row, worker=True) + rows.append(row) + audits.append(pg_publish.audit_row_for(row, request_id="", writer=SUBJECT)) + documents.append(document) + return Block(rows, audits, documents) + + +async def _expected_document( + args: argparse.Namespace, row: dict[str, Any], *, worker: bool = False +) -> dict[str, Any]: + from server.embedding import get_embedder + from server.llm import customer + from server.memory.domain.embed_text import embedding_texts + from server.memory.store.rows import parse_memory_value, search_document_row + + embedder = get_embedder() + raw = str(row["value"]) + excerpt, text, key_text = embedding_texts( + row["memory_type"], parse_memory_value(raw), raw, row["memory_key"] + ) + wanted = sorted({t for t in (text, key_text) if t}) + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + vectors = dict(zip(wanted, await embedder.embed_many(wanted), strict=True)) + from server.memory.store.conflict_resolver import OWED_MARK + from server.settings import settings + + return search_document_row( + row, + conflict_checked_at=OWED_MARK + if worker and settings.memory_conflict_resolver_enabled + else None, + embedding_model=embedder.model_name, + embedding=vectors.get(text, []), + key_embedding=vectors.get(key_text, [0.0] * embedder.dim), + key_text=key_text, + excerpt=excerpt, + ) + + +async def _stage(conn: Any, ch: Any, block: Block) -> None: + from server.memory.store import pg_memory + + await _debt(conn, 0) + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + fresh = await conn.fetchval( + "SELECT count(*) FROM loom.memory_current WHERE org=$1 AND namespace=$2 " + "AND (memory_id=ANY($3::uuid[]) OR memory_key=ANY($4::text[]))", + rows[0]["org"], + rows[0]["namespace"], + [uuid.UUID(str(row["memory_id"])) for row in rows], + [row["memory_key"] for row in rows], + ) + _require(fresh == 0, "fresh absent authority identity") + await _absent(ch, block) + committed = await pg_memory.commit_versions( + block.rows, + block.audits, + targets=pg_memory.MEMORY_ROW | pg_memory.SEARCH_DOC | pg_memory.AUDIT_ROW, + publisher_lease=False, + ) + _require( + len(committed.seqs) == len(block.rows) and all(t == 7 for t in committed.targets), + "staged delivery count/targets", + ) + block.seqs = committed.seqs + # The authority may canonicalize the full persisted row; retain its exact result. + _require( + len(committed.rows) == len(block.rows) and len(committed.audits) == len(block.audits), + "staged row/audit count", + ) + for expected, actual in zip(block.rows, committed.rows, strict=True): + _compare(expected, actual, tuple(expected), "committed full row") + for expected, actual in zip(block.audits, committed.audits, strict=True): + _compare(expected, actual, tuple(expected), "committed full audit") + await _debt(conn, len(block.rows)) + due = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries " + "WHERE targets=7 AND lease_until<=clock_timestamp() AND NOT held AND attempts=0" + ) + _require(due == len(block.rows), "exact fresh due deliveries") + + +async def _delivery( + args: argparse.Namespace, batch: int, *, emit: Callable[..., None], arm: int +) -> float: + from server.llm import customer + from server.memory.workers import pg_delivery + from server.settings import settings + + results: list[dict[str, Any]] = [] + _index = 0 + with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): + started = time.perf_counter_ns() + try: + for _index in range(BATCHES): + _require(settings.pg_delivery_batch_rows == batch, "unchanged delivery batch") + result = await pg_delivery.run_pass() + results.append(asdict(result)) + _require( + result.claimed == result.published == batch and result.failed == 0, + "real worker claimed/published/failure", + ) + except BaseException as error: + emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + raise + finally: + elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 + for result_index, outcome in enumerate(results): + emit("worker", arm=arm, index=result_index, **outcome) + _require(math.isfinite(elapsed) and elapsed > 0, "positive worker block elapsed") + rps = BATCHES * batch / elapsed + emit("delivery_block", arm=arm, elapsed_seconds=elapsed, rows=BATCHES * batch, rps=rps) + return rps + + +def _normal(value: Any) -> Any: + from datetime import datetime + + if isinstance(value, (bytes, bytearray)): + return hash_text(value) + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, str) and re.fullmatch(r"\d{4}-\d\d-\d\dT.*", value): + try: + value = datetime.fromisoformat(value) + except ValueError: + pass + if isinstance(value, datetime): + return value.replace(tzinfo=None).isoformat(timespec="milliseconds") + if isinstance(value, Mapping): + return {k: _normal(v) for k, v in value.items()} + if isinstance(value, (tuple, list)): + return [_normal(v) for v in value] + return value + + +def _compare( + expected: Mapping[str, Any], actual: Mapping[str, Any], fields: Sequence[str], phase: str +) -> None: + for field_name in fields: + left, right = expected.get(field_name), actual.get(field_name) + if field_name in ("embedding", "memory_key_embedding"): + _require( + isinstance(left, list) + and isinstance(right, list) + and len(left) == len(right) + and all( + math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) + for a, b in zip(left, right, strict=True) + ), + phase + " vector", + ) + else: + _require(_normal(left) == _normal(right), phase + " " + field_name) + + +async def _absent(ch: Any, block: Block) -> None: + for offset in range(0, len(block.rows), 100): + rows = block.rows[offset : offset + 100] + params = { + "org": rows[0]["org"], + "ns": rows[0]["namespace"], + "ids": [str(r["memory_id"]) for r in rows], + "keys": [r["memory_key"] for r in rows], + "hashes": [a["row_hash"] for a in block.audits[offset : offset + 100]], + } + for table, predicate in ( + ("memory", "memory_id IN {ids:Array(UUID)}"), + ("memory_embeddings", "memory_key IN {keys:Array(String)}"), + ("audit_log", "row_hash IN {hashes:Array(String)}"), + ): + final = "" if table == "audit_log" else " FINAL" + found = await ch.query_all( + f"SELECT count() AS n FROM loom.{table}{final} WHERE org = " + f"{{org:String}} AND namespace = {{ns:String}} AND {predicate}", + params, + ) + _require(found == [{"n": 0}], "fresh absent " + table) + + +async def _verify( + conn: Any, ch: Any, args: argparse.Namespace, identities: Sequence[RequestRow] | Block +) -> None: + block = identities if isinstance(identities, Block) else None + wanted = identities.rows if isinstance(identities, Block) else [asdict(r) for r in identities] + for offset in range(0, len(wanted), 100): + group = wanted[offset : offset + 100] + ids = [uuid.UUID(str(r["memory_id"])) for r in group] + stored = await conn.fetch( + "SELECT c.row::text AS head, r.row::text AS revision, " + "r.audit::text AS audit, r.seq, d.targets, d.attempts, " + "d.last_error FROM loom.memory_current c JOIN " + "loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT JOIN " + "loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(r.org,r.namespace,r.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[])", + args.org, + args.namespace, + ids, + ) + _require(len(stored) == len(group), "exact authority head/revision count") + by_id = {json.loads(r["head"])["memory_id"]: r for r in stored} + params = { + "org": args.org, + "ns": args.namespace, + "ids": [str(i) for i in ids], + "keys": [r.get("memory_key", r.get("key")) for r in group], + } + projected = await ch.query_all( + "SELECT * FROM loom.memory FINAL WHERE org={org:String} AND " + "namespace={ns:String} AND memory_id IN {ids:Array(UUID)}", + params, + ) + docs = await ch.query_all( + "SELECT * FROM loom.memory_embeddings FINAL WHERE org={org:String} " + "AND namespace={ns:String} AND memory_key IN {keys:Array(String)}", + params, + ) + _require(len(projected) == len(docs) == len(group), "exact projection/vector counts") + projections = {str(r["memory_id"]): r for r in projected} + documents = {r["memory_key"]: r for r in docs} + audit_hashes = [] + authority_audits = {} + for index, expected in enumerate(group): + id = str(expected["memory_id"]) + _require(id in by_id and id in projections, "authority/projection identity") + record = by_id[id] + _require( + record["targets"] in (None, 0) + and record["attempts"] in (None, 0) + and record["last_error"] in (None, ""), + "delivery settled without failure", + ) + head, revision, audit = ( + json.loads(record[name]) for name in ("head", "revision", "audit") + ) + _compare(head, revision, tuple(head), "authority full row") + if block: + _compare(expected, head, tuple(expected), "authority staged row") + _compare( + block.audits[offset + index], audit, tuple(audit), "authority staged audit" + ) + _require( + (args.org, args.namespace, record["seq"]) == block.seqs[offset + index], + "authority staged sequence", + ) + else: + key = str(expected["key"]) + _require( + head["memory_key"] == key + and json.loads(head["value"]) == _payload(key) + and head["version"] == expected["version"] + and head["content_sha256"] == expected["content_sha256"] + and head["status"] == "active", + "authority API payload/version", + ) + _compare(head, projections[id], tuple(head), "full memory projection") + key = str(head["memory_key"]) + _require(key in documents, "search document identity") + document = documents[key] + if block: + expected_document = block.documents[offset + index] + _compare( + expected_document, + document, + tuple(k for k in expected_document if k != "created_at"), + "full search document", + ) + else: + expected_document = await _expected_document(args, head) + _compare( + expected_document, + document, + tuple( + k + for k in expected_document + if k not in ("created_at", "conflict_checked_at") + ), + "full API search document", + ) + audit_hashes.append(audit["row_hash"]) + authority_audits[audit["row_hash"]] = audit + params["hashes"] = audit_hashes + audits = await ch.query_all( + "SELECT * FROM loom.audit_log WHERE org={org:String} AND " + "namespace={ns:String} AND row_hash IN {hashes:Array(String)}", + params, + ) + _require( + sorted(hash_text(r["row_hash"]) for r in audits) == sorted(audit_hashes), + "exact stored audit hashes", + ) + for audit in audits: + expected_audit = authority_audits[hash_text(audit["row_hash"])] + _compare(expected_audit, audit, tuple(expected_audit), "full stored audit") + await _debt(conn, 0) + + +async def _settle_api( + conn: Any, ch: Any, args: argparse.Namespace, rows: Sequence[RequestRow] +) -> None: + end = asyncio.get_running_loop().time() + SETTLE_S + while True: + pending = await conn.fetchval( + "SELECT count(*) FROM loom.memory_deliveries WHERE org=$1 AND " + "namespace=$2 AND targets<>0", + args.org, + args.namespace, + ) + if pending == 0: + break + _require(asyncio.get_running_loop().time() < end, "API publication settlement deadline") + await asyncio.sleep(0.1) + await _verify(conn, ch, args, rows) + + +async def _physical(conn: Any, ch: Any) -> dict[str, Any]: + from server.core.ch_names import database, telemetry_database + + pg = await conn.fetch( + "SELECT relname, n_live_tup AS estimated_rows, " + "pg_total_relation_size(relid) AS bytes FROM pg_stat_user_tables WHERE " + "schemaname='loom' ORDER BY relname" + ) + parts = await ch.query_all( + "SELECT database, table, sum(rows) AS rows, sum(bytes_on_disk) AS " + "bytes FROM system.parts WHERE active AND database IN " + "{dbs:Array(String)} GROUP BY database, table ORDER BY database, table", + {"dbs": [database(), telemetry_database()]}, + ) + exact = {} + for table in ("memory_current", "memory_revisions", "memory_deliveries", "audit_outbox"): + exact[table] = await conn.fetchval(f"SELECT count(*) FROM loom.{table}") + return { + "postgres_rows": exact, + "postgres_estimates": [dict(r) for r in pg], + "clickhouse_parts": parts, + } + + +async def _run( + args: argparse.Namespace, fixture: Fixture, artifact: Artifact, ch: Any +) -> dict[str, Any]: + from server.core.ch import set_query_tag + from server.settings import settings + + set_query_tag("bench.cell_trigger_gate") + batch = settings.pg_delivery_batch_rows + conn = await asyncpg.connect( + args.pg_dsn, + timeout=IO_S, + command_timeout=IO_S, + server_settings={"loom.tenancy_checked": "on"}, + ) + triggers = None + samples = [] try: - times = [] - for _ in range(INSERTS): - t = time.perf_counter() - await conn.execute( - "INSERT INTO loom.scope_generations (org, scope) " - "SELECT $1, $2 || g || '-' || $3 FROM generate_series(1, 1000) g", - args.org, - args.scope_prefix, - uuid.uuid4().hex[:8], + async with asyncio.timeout(IO_S): + await _proof(conn, ch, args, fixture, native=True) + triggers = await Triggers.capture(conn) + for arm, on in enumerate(_arm_order()): + async with asyncio.timeout(IO_S): + proof = await _proof(conn, ch, args, fixture, native=True) + await triggers.set(conn, on) + artifact.emit( + "arm_proof", + arm=arm, + on=on, + **proof, + tables=triggers.tables, + invariant_sha256=triggers.invariant, + org_triggers_sha256=triggers.org, + ) + api = await _memory_set(args, arm=arm, emit=artifact.emit) + async with asyncio.timeout(SETTLE_S): + await _settle_api(conn, ch, args, api) + block = await _build_block(args, batch) + async with asyncio.timeout(SETTLE_S): + await _stage(conn, ch, block) + rps = await _delivery(args, batch, emit=artifact.emit, arm=arm) + async with asyncio.timeout(SETTLE_S): + await _verify(conn, ch, args, block) + artifact.emit( + "settled", + arm=arm, + api_rows=len(api), + delivery_rows=len(block.rows), + physical=await _physical(conn, ch), + ) + latencies = [r.elapsed_ms for r in api if r.measured] + samples.append(Samples(on, latencies, rps)) + artifact.emit( + "arm", + arm=arm, + on=on, + p50_ms=_quantile(latencies, 0.5), + p99_ms=_quantile(latencies, 0.99), + delivery_rps=rps, ) - times.append((time.perf_counter() - t) * 1000) - return statistics.median(times) + from server.core.background import drain_task_sets, flush_registered_buffers + + _require(await drain_task_sets(timeout_s=IO_S), "driver descendant drain") + _require(await flush_registered_buffers(), "driver buffer drain") + summary = _statistics(samples) + artifact.emit("provisional_statistics", **summary) + return summary finally: - await conn.close() - - -async def main() -> None: - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - p.add_argument("--base-url", required=True) - p.add_argument("--pg-dsn", required=True) - p.add_argument("--token", required=True) - p.add_argument("--org", required=True) - p.add_argument("--namespace", required=True) - args = p.parse_args() - args.scope_prefix = f"gate-{uuid.uuid4().hex}-" + prior = sys.exception() + cleanup = asyncio.create_task( + _cleanup(triggers, conn, artifact), name="trigger-gate-cleanup" + ) + interrupted = False + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + interrupted = True + except BaseException: # noqa: BLE001 — preserve an earlier operation failure + break + try: + cleanup.result() + except BaseException as error: + try: + artifact.emit("cleanup_error", error_type=type(error).__name__) + except (OSError, ValueError): + if prior is None: + raise + if prior is None: + raise + if interrupted and prior is None: + raise asyncio.CancelledError + + +async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: try: - for _ in range(ROUNDS): - for on in (True, False): - await _triggers(args.pg_dsn, on) - p50, p99 = await _memory_set(args) - marked, unmarked = await _batch(args, True), await _batch(args, False) - print( - f"triggers {'on ' if on else 'off'}: memory.set p50 {p50:.1f} ms " - f"p99 {p99:.1f} ms; 1,000-row insert {marked:.1f} ms marked, " - f"{unmarked:.1f} ms unmarked", - flush=True, - ) + if triggers is not None: + await _restore(triggers, conn) + artifact.emit("triggers_restored", verified=True) finally: - await _triggers(args.pg_dsn, True) - await _clean(args) + prior = sys.exception() + try: + await _close_pg(conn) + except BaseException: + if prior is None: + raise + artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + + +async def _restore(triggers: Triggers, conn: Any) -> None: + async with asyncio.timeout(RESTORE_S): + await triggers.restore(conn) -async def _clean(args: argparse.Namespace) -> None: - """Delete the rows the inserts wrote (the `memory.set` calls' rows stay the org's).""" - conn = await asyncpg.connect(args.pg_dsn, server_settings={"loom.tenancy_checked": "on"}) +def main() -> int: + from server.cli import _ch + from server.settings import settings + from server.wiring import wire + + parser = argparse.ArgumentParser(description=__doc__) + for name in ("base-url", "pg-dsn", "token", "org", "namespace", "fixture-manifest", "output"): + parser.add_argument("--" + name, required=True) + args = parser.parse_args() + artifact = None + verdict: dict[str, Any] = {"verdict": "INVALID"} + phase = "local_manifest" try: - await conn.execute( - "DELETE FROM loom.scope_generations WHERE org = $1 AND starts_with(scope, $2)", - args.org, - args.scope_prefix, + fixture = Fixture.read(args.fixture_manifest) + total = fixture.validate(args, settings) + source = _source(fixture) + artifact = Artifact(args.output) + artifact.emit( + "configuration", + manifest=asdict(fixture), + source=source, + manifest_sha256=_digest(asdict(fixture)), + calls=CALLS, + warmup=WARMUP, + rounds=ROUNDS, + batches=BATCHES, + batch_rows=settings.pg_delivery_batch_rows, + logical_keys=total, + value_bytes=VALUE_BYTES, + order=_arm_order(), + driver_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), ) + phase = "readonly_preflight" + asyncio.run(_preflight(args, fixture)) + phase = "native_driver" + wire() + + async def run(ch: Any) -> dict[str, Any]: + return await _run(args, fixture, artifact, ch) + + summary = _ch.run_connected(run, orgs=(args.org,)) + artifact.emit("driver_cleanup", native_returned=True, outer_database_cleanup_required=True) + verdict = { + "verdict": "PASS" + if summary["latency_within_limit"] and summary["throughput_within_limit"] + else "FAIL", + "p99_delta_ms": summary["p99_delta_ms"], + "throughput_loss": summary["throughput_loss"], + "outer_database_cleanup_required": True, + } + except BaseException as error: # noqa: BLE001 — every failed run must emit INVALID + # Never serialize exception text: clients may include credentials or values. + reason = str(error) if isinstance(error, Invalid) else "operation refused" + verdict = { + "verdict": "INVALID", + "phase": phase, + "reason": reason, + "error_type": type(error).__name__, + } + if artifact is not None: + try: + artifact.emit( + "invalid", phase=phase, reason=reason, error_type=type(error).__name__ + ) + except (OSError, ValueError): + pass finally: - await conn.close() + if artifact is not None: + try: + artifact.close() + except (OSError, ValueError) as error: + verdict = { + "verdict": "INVALID", + "error_type": type(error).__name__, + "phase": "artifact_close", + } + print(json.dumps(verdict, sort_keys=True, allow_nan=False), flush=True) + return 0 if verdict["verdict"] == "PASS" else 1 if __name__ == "__main__": - asyncio.run(main()) + raise SystemExit(main()) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 89f32cddde..a27f9560f3 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -1,53 +1,971 @@ -"""The trigger gate deletes only the scope rows its own batch inserts wrote.""" +"""Offline proofs for the fixed cell trigger timing protocol; no fixture is opened.""" from __future__ import annotations import argparse -import sqlite3 -from typing import Any +import asyncio +import dataclasses +import json +import uuid +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +import httpx import pytest from bench import cell_trigger_gate as gate +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected real I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(gate.httpx.AsyncClient, "send", forbidden) + + +def args(**overrides): + return argparse.Namespace( + **{ + "base_url": "http://127.0.0.1:17932", + "pg_dsn": "postgresql://u:secret@127.0.0.1:15790/gate_cell", + "token": "secret-token", + "org": "org", + "namespace": "ns", + **overrides, + } + ) + + +def fixture_config(): + config = SimpleNamespace( + pg_dsn=args().pg_dsn, + ch_host="127.0.0.1", + ch_port=18790, + ch_database="gate_cell", + ch_telemetry_database="gate_cell_t", + cell="gate-cell", + background_workers="off", + storage_authority="postgres", + embedding_provider="fake", + llm_provider="fake", + build_commit="a" * 40, + pg_delivery_batch_rows=500, + ) + config.model_dump = lambda **_: {k: v for k, v in vars(config).items() if k != "model_dump"} + manifest = gate.Fixture( + base_url=args().base_url, + pg_host="127.0.0.1", + pg_port=15790, + pg_database="gate_cell", + ch_host=config.ch_host, + ch_port=config.ch_port, + ch_database=config.ch_database, + ch_telemetry_database=config.ch_telemetry_database, + cell=config.cell, + org="org", + namespace="ns", + deployment_id="deployment", + epoch=1, + instance_id="instance", + build_commit=config.build_commit, + source_tree="b" * 40, + launch_manifest_sha256="c" * 64, + ownership_receipt_sha256="d" * 64, + configuration_sha256=gate._digest(config.model_dump()), + background_workers="off", + controller_excluded=True, + pg_internal_port=5432, + ) + return manifest, config + + +def test_gate_keeps_the_original_api_sample_contract(): + assert (gate.CALLS, gate.WARMUP, gate.ROUNDS, gate.BATCHES) == (2000, 50, 5, 5) + assert not hasattr(gate, "_batch") and not hasattr(gate, "_clean") + + +@pytest.mark.parametrize(("batch", "expected"), [(500, 45500), (1590, 100000)]) +def test_complete_fixture_count_uses_the_unchanged_delivery_batch(batch, expected): + assert gate._planned_keys(batch) == expected + + +@pytest.mark.parametrize("batch", [1591, 0, -1, 1.5, True]) +def test_invalid_batch_refuses_before_loading(batch): + with pytest.raises(ValueError): + gate._planned_keys(batch) + + +def test_arm_order_alternates_pair_order_and_has_five_of_each(): + order = gate._arm_order() + assert order == (True, False, False, True, True, False, False, True, True, False) + assert order.count(True) == order.count(False) == 5 + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost:1234", + "http://example.com:1234", + "http://10.0.0.1:1234", + "http://127.0.0.1", + "http://127.0.0.1:1234/other", + "http://u:pw@127.0.0.1:1234", + "http://127.0.0.1:1234?x=1", + "http://127.0.0.1:1234#fragment", + ], +) +def test_http_endpoint_refuses_ambiguous_or_remote_targets(url): + with pytest.raises(gate.Invalid): + gate._endpoint(url) + + +@pytest.mark.parametrize( + "dsn", + [ + "postgresql://u@localhost:15790/scratch", + "postgresql://u@127.0.0.1:15790/loom", + "postgresql://u@127.0.0.1:15790/scratch?host=remote", + "postgresql:///scratch?service=other", + "postgresql://u@127.0.0.1,127.0.0.2:15790/scratch", + "postgresql://u@127.0.0.1:15790/scratch?options=-cdefault_transaction_read_only=off", + ], +) +def test_pg_endpoint_refuses_overrides_defaults_and_alternate_targets(dsn): + with pytest.raises(gate.Invalid): + gate._endpoint(dsn, pg=True) + + +def test_manifest_binds_published_port_without_conflating_internal_port(): + manifest, config = fixture_config() + assert manifest.pg_port == 15790 and manifest.pg_internal_port == 5432 + assert manifest.validate(args(), config) == 45500 + with pytest.raises(gate.Invalid, match="manifest PG endpoint"): + dataclasses.replace(manifest, pg_port=5432).validate(args(), config) + + +@pytest.mark.parametrize( + "change", + [ + {"ch_host": "10.0.0.1"}, + {"ch_database": "loom"}, + {"cell": "other"}, + {"background_workers": "on"}, + {"llm_provider": "openai"}, + {"embedding_provider": "local"}, + {"pg_delivery_batch_rows": 1591}, + ], +) +def test_configuration_drift_and_unsafe_settings_refuse(change): + manifest, config = fixture_config() + for k, v in change.items(): + setattr(config, k, v) + with pytest.raises(gate.Invalid): + manifest.validate(args(), config) + + +def test_artifact_never_overwrites_existing_file_or_symlink(tmp_path): + original = tmp_path / "old.jsonl" + original.write_text("preserve") + with pytest.raises(FileExistsError): + gate.Artifact(str(original)) + link = tmp_path / "link" + link.symlink_to(original) + with pytest.raises(FileExistsError): + gate.Artifact(str(link)) + assert original.read_text() == "preserve" + fresh = tmp_path / "fresh" + artifact = gate.Artifact(str(fresh)) + artifact.emit("test", n=1) + artifact.close() + assert fresh.stat().st_mode & 0o777 == 0o600 + assert json.loads(fresh.read_text()) == {"kind": "test", "n": 1} + + @pytest.mark.asyncio -async def test_batch_cleanup_preserves_preexisting_gate_scopes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - db = sqlite3.connect(":memory:") - db.execute("ATTACH DATABASE ':memory:' AS loom") - db.execute("CREATE TABLE loom.scope_generations (org text, scope text)") - db.create_function("starts_with", 2, lambda value, prefix: value.startswith(prefix)) - prefix = "gate-own-run-" - sentinel = [("org", "gate-existing"), ("other", prefix + "existing")] - db.executemany("INSERT INTO loom.scope_generations VALUES (?, ?)", sentinel) - - class _Conn: - async def execute(self, sql: str, *args: Any) -> None: - sql = sql.replace( - "FROM generate_series(1, 1000) g", - "FROM (WITH RECURSIVE seq(g) AS (VALUES(1) UNION ALL " - "SELECT g + 1 FROM seq WHERE g < 1000) SELECT g FROM seq)", +async def test_actual_api_loop_sends_unique_four_kib_values_and_full_sample(monkeypatch): + sent, outcomes = [], [] + + class Client: + def __init__(self, **kw): + assert kw["follow_redirects"] is False and kw["trust_env"] is False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + sent.append(json) + return httpx.Response( + 200, + json={ + "memory_id": str(uuid.uuid4()), + "version": len(sent), + "admission": "ready", + "status": "active", + "content_sha256": gate.sha256_value(json["value"]), + "memory_key": json["memory_key"], + "namespace": json["namespace"], + }, + request=httpx.Request("POST", url), ) - db.execute(sql, {str(i): value for i, value in enumerate(args, 1)}) - async def close(self) -> None: + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + rows = await gate._memory_set( + args(), emit=lambda kind, **fields: outcomes.append((kind, fields)) + ) + assert len(sent) == len(rows) == len(outcomes) == 2050 + assert ( + len({body["memory_key"] for body in sent}) == len({body["value"] for body in sent}) == 2050 + ) + assert all(len(body["value"].encode("utf-8")) == 4096 for body in sent) + assert sum(row.measured for row in rows) == 2000 + assert all("value" not in fields and "token" not in fields for _, fields in outcomes) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status,changes", + [ + (302, {}), + (503, {}), + (200, {"status": "duplicate"}), + (200, {"noop": True}), + (200, {"content_sha256": "wrong"}), + (200, {"version": 0}), + ], +) +async def test_api_refusal_is_recorded_once_and_never_enters_samples(monkeypatch, status, changes): + events = [] + + class Client: + def __init__(self, **_): pass - async def connect(*_a: Any, **_kw: Any) -> Any: - return _Conn() + async def __aenter__(self): + return self - monkeypatch.setattr(gate.asyncpg, "connect", connect) - monkeypatch.setattr(gate, "INSERTS", 1) - args = argparse.Namespace(pg_dsn="unused", org="org", scope_prefix=prefix) - try: - await gate._batch(args, True) - assert db.execute("SELECT count(*) FROM loom.scope_generations").fetchone()[0] == 1002 - await gate._clean(args) - assert ( - db.execute("SELECT org, scope FROM loom.scope_generations ORDER BY org").fetchall() - == sentinel + async def __aexit__(self, *_): + return None + + async def post(self, url, *, json): + body = { + "memory_id": str(uuid.uuid4()), + "version": 1, + "status": "active", + "admission": "ready", + "memory_key": json["memory_key"], + "namespace": json["namespace"], + "content_sha256": gate.sha256_value(json["value"]), + **changes, + } + return httpx.Response(status, json=body, request=httpx.Request("POST", url)) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda *a, **kw: events.append((a, kw))) + assert len(events) == 1 and events[0][0] == ("request_error",) + assert "secret-token" not in str(events) and "value" not in events[0][1] + + +def samples(on_p99, off_p99, on_rps=90.1, off_rps=100): + on, off = iter(on_p99), iter(off_p99) + return [ + gate.Samples(arm, [next(on) if arm else next(off)] * gate.CALLS, on_rps if arm else off_rps) + for arm in gate._arm_order() + ] + + +def test_statistics_use_difference_of_arm_medians_and_throughput_not_time(): + result = gate._statistics(samples([2, 12, 22, 32, 102], [1, 6, 51, 61, 71])) + assert result["p99_delta_ms"] == -29 + assert result["paired_p99_differences_ms"] == [1, 6, -29, -29, 31] + assert result["latency_within_limit"] is True + # Time increases 10.99%, but the original rows/second gate passes at 9.9% loss. + assert result["throughput_loss"] == pytest.approx(0.099) + assert result["throughput_within_limit"] is True + failed = gate._statistics(samples([20] * 5, [1] * 5, 89, 100)) + assert failed["latency_within_limit"] is failed["throughput_within_limit"] is False + assert gate._quantile(list(range(2000)), 0.99) == 1979 + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) +def test_invalid_throughput_cannot_become_passing_number(value): + with pytest.raises(gate.Invalid): + gate._statistics(samples([1] * 5, [1] * 5, value)) + + +@pytest.mark.asyncio +async def test_actual_production_worker_body_runs_five_times_with_unchanged_batch(monkeypatch): + from server.core import idempotency, pg + from server.llm import customer + from server.memory.app import pg_delivery as publish + from server.memory.store import pg_memory + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(worker, "_read_backlog", AsyncMock()) + claim = AsyncMock(return_value=[object(), object()]) + monkeypatch.setattr(pg_memory, "claim_due", claim) + attribution = [] + + async def publish_claims(claims): + attribution.append(customer._attribution.get()) + assert len(claims) == 2 + return 2, 0 + + monkeypatch.setattr(publish, "publish_claims", publish_claims) + for module, name in ( + (idempotency, "sweep_expired"), + (pg_memory, "sweep_erasure_evidence"), + (pg_memory, "sweep_settled"), + (publish, "publish_audit_outbox"), + ): + monkeypatch.setattr(module, name, AsyncMock(return_value=0)) + times = iter([0, 2_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(times)) + events = [] + assert ( + await gate._delivery(args(), 2, emit=lambda kind, **kw: events.append((kind, kw)), arm=0) + == 5 + ) + assert claim.await_count == 5 and all(call.args == (2,) for call in claim.await_args_list) + assert len(attribution) == 5 + assert all( + a.org == "org" and a.namespace == "ns" and a.subject == gate.SUBJECT for a in attribution + ) + assert [kind for kind, _ in events] == ["worker"] * 5 + ["delivery_block"] + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result", + [ + {"claimed": 1, "published": 1}, + {"claimed": 2, "published": 1}, + {"claimed": 2, "published": 2, "failed": 1}, + ], +) +async def test_worker_count_failure_invalidates_block_without_retries(monkeypatch, result): + from server.memory.workers import pg_delivery as worker + from server.settings import settings + + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 2) + run = AsyncMock(return_value=worker.DeliveryResult(**result)) + monkeypatch.setattr(worker, "run_pass", run) + with pytest.raises(gate.Invalid): + await gate._delivery(args(), 2, emit=lambda *_a, **_kw: None, arm=0) + assert run.await_count == 1 + + +@pytest.mark.asyncio +async def test_canonical_full_rows_audits_and_expected_vectors_use_real_fake_embedder(monkeypatch): + import server.embedding + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + embedder = FakeNGramEmbedder(dim=16) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: embedder) + # Keep actual require(): refresh its real switch read through a fake store edge. + import logging + + from server.core.polled_copy import PolledCopy + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + block = await gate._build_block(args(), 1) + assert len(block.rows) == len(block.audits) == len(block.documents) == 5 + for row, audit, doc in zip(block.rows, block.audits, block.documents, strict=True): + assert set(MEMORY_PERSISTED_COLS) <= row.keys() + assert len(json.loads(row["value"]).encode()) == 4096 + assert row["actor"] == gate.SUBJECT and row["source_class"] == "asserted" + assert row["content_sha256"] == gate.sha256_value(json.loads(row["value"])) + assert audit["event_type"] == "memory.revision" and len(audit["row_hash"]) == 64 + assert row["value"] not in audit["subject"] + assert len(doc["embedding"]) == doc["embedding_dim"] == 16 + assert doc["content_sha256"] == row["content_sha256"] + assert pg_memory.decode_row(pg_memory.encode_row(row)) == row + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_stage_calls_real_commit_interface_with_due_fresh_targets(monkeypatch): + from server.memory.store import pg_memory + + block = gate.Block( + [{"memory_id": str(uuid.uuid4()), "org": "org", "namespace": "ns", "memory_key": "key"}], + [{"row_hash": "hash"}], + [], + ) + debt = AsyncMock() + absent = AsyncMock() + monkeypatch.setattr(gate, "_debt", debt) + monkeypatch.setattr(gate, "_absent", absent) + committed = pg_memory.Committed( + rows=block.rows, audits=block.audits, seqs=[("org", "ns", 1)], targets=[7] + ) + commit = AsyncMock(return_value=committed) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + conn = SimpleNamespace(fetchval=AsyncMock(side_effect=[0, 1])) + await gate._stage(conn, object(), block) + assert commit.await_args.kwargs == {"targets": 7, "publisher_lease": False} + assert block.seqs == [("org", "ns", 1)] + assert [call.args[1] for call in debt.await_args_list] == [0, 1] + absent.assert_awaited_once() + + +@pytest.mark.parametrize( + ("field", "expected", "actual"), + [ + ("value", "right", "wrong"), + ("version", 2, 1), + ("content_sha256", "a", "b"), + ("embedding", [1.0, 0.0], [0.0, 1.0]), + ("embedding", [1.0], []), + ("row_hash", "good", "bad"), + ], +) +def test_stored_payload_vector_version_and_audit_mismatch_refuse(field, expected, actual): + with pytest.raises(gate.Invalid): + gate._compare({field: expected}, {field: actual}, [field], "stored") + + +class TriggerConn: + def __init__(self): + self.on = True + self.drift = False + self.statements = [] + + @asynccontextmanager + async def transaction(self): + yield + + async def execute(self, sql): + self.statements.append(sql) + if sql.startswith("DROP TRIGGER"): + self.on = False + elif sql.startswith("CREATE TRIGGER"): + self.on = True + + +@pytest.mark.asyncio +async def test_captured_trigger_state_allows_off_to_off_and_restores_exact_tables(monkeypatch): + conn = TriggerConn() + + async def snapshot(c): + return ( + ("memory_current", "tenancy"), + "drift" if c.drift else "invariant", + "org" if c.on else gate._digest([]), + ) + + monkeypatch.setattr(gate.Triggers, "snapshot", staticmethod(snapshot)) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + triggers = await gate.Triggers.capture(conn) + await triggers.set(conn, False) + await triggers.set(conn, False) + assert triggers.on is False + await triggers.restore(conn) + assert triggers.on is True + assert conn.statements and all('ON loom."memory_current"' in sql for sql in conn.statements) + conn.drift = True + with pytest.raises(gate.Invalid, match="drift"): + await triggers.set(conn, False) + + +class ProofPG: + def __init__(self, fault=""): + self.fault = fault + self.closed = False + self.sql = [] + + async def fetchrow(self, sql, *params): + self.sql.append(sql) + if "current_database()" in sql: + return { + "db": "wrong" if self.fault == "database" else "gate_cell", + "port": 5432, + "recovery": self.fault == "recovery", + } + if "FROM loom.authority" in sql: + return { + "authority": "postgres", + "deployment_id": "wrong" if self.fault == "pair" else "deployment", + "epoch": 1, + } + if "FROM loom.cell_first" in sql: + return None if self.fault == "partial" else {"cell": "gate-cell", "is_first": False} + if "FROM loom.cell_self" in sql: + return { + "current_fp": json.dumps({"v": 1, "fields": {}}), + "next_fp": {} if self.fault == "transition" else None, + } + if "AS pending" in sql: + return {"pending": int(self.fault == "debt"), "audits": 0, "errors": 0} + raise AssertionError(sql) + + async def fetchval(self, sql, *params): + self.sql.append(sql) + if "FROM loom.tenancy_mode" in sql: + return "recording" if self.fault == "recording" else "enforced" + if "FROM loom.tenancy WHERE" in sql: + return self.fault != "held" + raise AssertionError(sql) + + async def close(self, **kwargs): + self.closed = True + + +class ProofCH: + def __init__(self, fault=""): + self.fault = fault + self.closed = False + self.sql = [] + + async def query_all(self, sql, parameters=None): + self.sql.append(sql) + if "currentDatabase()" in sql: + return [{"db": "wrong" if self.fault == "ch_database" else "gate_cell"}] + if "FROM loom.storage_authority" in sql: + return [ + { + "authority": "postgres", + "deployment_id": "wrong" if self.fault == "ch_pair" else "deployment", + "epoch": 1, + } + ] + if "FROM loom.provider_configuration" in sql: + return [{"n": int(self.fault == "saved_provider")}] + if "FROM loom.runtime_overrides" in sql: + return [{"n": int(self.fault == "overrides")}] + if "FROM loom.cell_heartbeats" in sql: + return [ + { + "cell": "gate-cell", + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + {"serves": True, "marked": self.fault != "fingerprint"} + ), + } + ] + raise AssertionError(sql) + + async def close(self): + self.closed = True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "database", + "recovery", + "pair", + "partial", + "recording", + "held", + "transition", + "debt", + "ch_database", + "ch_pair", + "saved_provider", + "overrides", + "fingerprint", + "health", + "competing_worker", + ], +) +async def test_read_only_preflight_faults_refuse_before_native_setup(monkeypatch, fault): + import server.settings + from server.cli import _ch + from server.core import cell, instances + + fixture, config = fixture_config() + pg, ch = ProofPG(fault), ProofCH(fault) + connections = [] + + async def connect(dsn, **kwargs): + connections.append(kwargs) + assert kwargs["server_settings"] == {"default_transaction_read_only": "on"} + return pg + + async def close(conn): + await conn.close() + + async def live(*_args, **_kw): + return ( + [] + if fault == "competing_worker" + else [{"instance_id": "instance", "build_commit": "a" * 40, "workers_lease": ""}] ) - finally: - db.close() + + async def health(*_args): + if fault == "health": + raise gate.Invalid("health source") + + native = AsyncMock(side_effect=AssertionError("native setup before proof")) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate.asyncpg, "connect", connect) + monkeypatch.setattr(gate.ReadOnlyCH, "open", AsyncMock(return_value=ch)) + monkeypatch.setattr(gate, "_close_pg", close) + monkeypatch.setattr(gate, "_health", health) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(instances, "read_live", live) + monkeypatch.setattr(cell, "mine", lambda: {}) + monkeypatch.setattr(cell, "matches", lambda mine, value: value is not None) + monkeypatch.setattr(_ch, "run_connected", native) + with pytest.raises(gate.Invalid): + await gate._preflight(args(), fixture) + assert pg.closed and ch.closed + native.assert_not_called() + assert connections and all(sql.startswith("SELECT ") for sql in pg.sql + ch.sql) + assert not any("model_transitions" in sql for sql in ch.sql) + + +@pytest.mark.asyncio +async def test_readonly_ch_startup_restricts_connection_before_first_query(monkeypatch): + client = SimpleNamespace(close=AsyncMock()) + opened = AsyncMock(return_value=client) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", opened) + ch = await gate.ReadOnlyCH.open() + assert opened.await_args.kwargs["settings"]["readonly"] == 1 + await ch.close() + client.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_health_never_follows_redirects(monkeypatch): + fixture, _ = fixture_config() + + class Client: + def __init__(self, **kwargs): + assert kwargs["follow_redirects"] is False and kwargs["trust_env"] is False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return None + + async def get(self, url): + return httpx.Response( + 302, + headers={"Location": "http://external.invalid"}, + request=httpx.Request("GET", url), + ) + + monkeypatch.setattr(gate.httpx, "AsyncClient", Client) + with pytest.raises(gate.Invalid, match="health HTTP status"): + await gate._health(args(), fixture) + + +@pytest.mark.asyncio +async def test_restore_preserves_primary_failure_and_survives_second_cancellation(monkeypatch): + from server.settings import settings + + fixture, _ = fixture_config() + conn = ProofPG() + restoring, release = asyncio.Event(), asyncio.Event() + failure = ValueError("primary-secret") + events = [] + + class T: + tables = ("memory_current",) + invariant = org = "digest" + + async def set(self, *_): + pass + + async def restore(*_): + restoring.set() + await release.wait() + + async def close(c): + c.closed = True + + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=conn)) + monkeypatch.setattr(gate.Triggers, "capture", AsyncMock(return_value=T())) + monkeypatch.setattr(gate, "_proof", AsyncMock(return_value={})) + monkeypatch.setattr(gate, "_memory_set", AsyncMock(side_effect=failure)) + monkeypatch.setattr(gate, "_restore", restore) + monkeypatch.setattr(gate, "_close_pg", close) + artifact = SimpleNamespace(emit=lambda kind, **kw: events.append(kind)) + task = asyncio.create_task(gate._run(args(), fixture, artifact, object())) + await restoring.wait() + task.cancel() + await asyncio.sleep(0) + task.cancel() + release.set() + with pytest.raises(ValueError) as raised: + await task + assert raised.value is failure and conn.closed + assert "triggers_restored" in events + assert settings.pg_delivery_batch_rows > 0 + + +@pytest.mark.parametrize("cleanup_failure", [False, True]) +def test_final_verdict_follows_native_cleanup_and_artifact_close( + monkeypatch, tmp_path, capsys, cleanup_failure +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "result.jsonl" + arguments = args() + argv = ["driver"] + for name, value in { + **vars(arguments), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + order = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: order.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + + def native(*_a, **_kw): + order.append("native_cleanup") + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + monkeypatch.setattr(_ch, "run_connected", native) + + def sync(_fd): + assert capsys.readouterr().out == "" + assert order == ["wire", "native_cleanup"] + if cleanup_failure: + raise OSError("secret-token") + + monkeypatch.setattr(gate.os, "fsync", sync) + assert gate.main() == int(cleanup_failure) + final = capsys.readouterr().out + assert "secret-token" not in final + assert json.loads(final)["verdict"] == ("INVALID" if cleanup_failure else "PASS") + assert '"verdict": "PASS"' not in output.read_text() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "projection_value", + "projection_version", + "vector", + "audit_hash", + "audit_subject", + "missing_document", + "targets", + "attempts", + "head", + ], +) +async def test_composed_stored_verification_compares_authority_projection_vectors_and_audits( + monkeypatch, fault +): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 1 if fault == "targets" else 0, + "attempts": 1 if fault == "attempts" else 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [dict(row) for row in block.rows] + documents = [dict(row) for row in block.documents] + audits = [dict(row) for row in block.audits] + if fault == "projection_value": + projections[0]["value"] = '"wrong"' + if fault == "projection_version": + projections[0]["version"] -= 1 + if fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + if fault == "audit_hash": + audits[0]["row_hash"] = "bad" + if fault == "audit_subject": + audits[0]["subject"] = "wrong" + if fault == "missing_document": + documents.pop() + if fault == "head": + records.pop() + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid): + await gate._verify(conn, ch, args(), block) + else: + await gate._verify(conn, ch, args(), block) + assert all( + "org=" in call.args[0] and "namespace=" in call.args[0] + for call in conn.fetch.await_args_list + ) + + +@pytest.mark.asyncio +async def test_real_customer_policy_can_refuse_expected_vectors_without_bypass(monkeypatch): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy( + "switch", AsyncMock(return_value=frozenset({"org"})), logging.getLogger(__name__) + ), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + with pytest.raises(customer.ModelCallsDisabled): + await gate._build_block(args(), 1) + assert customer._attribution.get() is None + + +@pytest.mark.asyncio +async def test_partial_trigger_toggle_is_restored_on_original_failure(monkeypatch): + conn = TriggerConn() + original = ValueError("DDL failed") + writes = 0 + + async def execute(sql): + nonlocal writes + writes += 1 + if writes == 2: + raise original + conn.on = sql.startswith("CREATE TRIGGER") + + async def snapshot(c): + return ("memory_current", "tenancy"), "invariant", "org" if c.on else gate._digest([]) + + monkeypatch.setattr(conn, "execute", execute) + monkeypatch.setattr(gate.Triggers, "snapshot", staticmethod(snapshot)) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + triggers = await gate.Triggers.capture(conn) + with pytest.raises(ValueError) as raised: + try: + await triggers.set(conn, False) + finally: + await triggers.restore(conn) + assert raised.value is original and conn.on and triggers.on + + +def test_manifest_unknown_fields_and_credentials_refuse(tmp_path): + fixture, _ = fixture_config() + path = tmp_path / "manifest" + path.write_text(json.dumps({**dataclasses.asdict(fixture), "password": "secret"})) + with pytest.raises(gate.Invalid, match="fixture manifest"): + gate.Fixture.read(str(path)) + + +@pytest.mark.parametrize("fault", ["", "head", "tree", "dirty", "refused"]) +def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fault): + fixture, _ = fixture_config() + calls = [] + + def git(argv, **kwargs): + calls.append(argv) + assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False + assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert "shell" not in kwargs + if argv[1:] == ["rev-parse", "HEAD"]: + value = "wrong" if fault == "head" else fixture.build_commit + elif argv[1:] == ["rev-parse", "HEAD^{tree}"]: + value = "wrong" if fault == "tree" else fixture.source_tree + else: + assert argv[1:] == ["status", "--porcelain"] + value = " M file" if fault == "dirty" else "" + return SimpleNamespace(returncode=int(fault == "refused"), stdout=value) + + monkeypatch.setattr(gate.subprocess, "run", git) + if fault: + with pytest.raises(gate.Invalid): + gate._source(fixture) + else: + assert gate._source(fixture) == { + "head": fixture.build_commit, + "tree": fixture.source_tree, + "clean": True, + } + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_restoration_has_finite_deadline_and_closes_owned_connection(monkeypatch): + conn = ProofPG() + + class Stuck: + async def restore(self, _): + await asyncio.Event().wait() + + monkeypatch.setattr(gate, "RESTORE_S", 0.01) + + async def close(owned): + assert owned is conn + await conn.close() + + monkeypatch.setattr(gate, "_close_pg", close) + artifact = SimpleNamespace(emit=lambda *_a, **_kw: None) + with pytest.raises(TimeoutError): + await gate._cleanup(Stuck(), conn, artifact) + assert conn.closed From 2e1e1ae3d8f6689973fbe05c3565762c48d7c364 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:20:47 +0000 Subject: [PATCH 058/181] Validate cell update targets against their registration Use the same registration checks for attach and configuration targets. Refuse changed stores or hosts, unsupported SCIM and fingerprints that cannot fit a live placement before any reservation or local acceptance change. Signed-off-by: zlareb1 --- server/cli/cell_update_config.py | 19 ++++ server/router/directory.py | 36 +++----- server/router/fingerprints.py | 28 +++++- tests/unit/test_cell_update_target.py | 122 ++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 27 deletions(-) create mode 100644 tests/unit/test_cell_update_target.py diff --git a/server/cli/cell_update_config.py b/server/cli/cell_update_config.py index a6303210db..9a83df9607 100644 --- a/server/cli/cell_update_config.py +++ b/server/cli/cell_update_config.py @@ -8,6 +8,9 @@ from pydantic import ValidationError from pydantic_settings import BaseSettings, PydanticBaseSettingsSource +from server.code.domain.index import repos +from server.core import cell +from server.router import fingerprints from server.settings import Settings from server.settings._file import SettingsFileError, read_settings_file from server.settings._live import coerce @@ -51,3 +54,19 @@ def read_target(path: Path, current: Settings) -> Settings: raise SettingsFileError( f"{path}: invalid target configuration; check the setting constraints" ) from None + + +def target_fingerprint( + target: Settings, deployment_id: str, registered: dict[str, Any] +) -> dict[str, Any]: + """Validate the candidate without admitting it or changing process settings.""" + value = cell.as_value( + cell.fingerprint_for(target, deployment_id, repos.operator_orgs_for(target)) + ) + if reason := fingerprints.registration_refusal(value, registered): + raise ValueError(reason) + if value["fields"]["scim"]: + raise ValueError("configuration update requires per-cell SCIM before enabling SCIM") + if reason := fingerprints.placement_refusal(registered["cell"], "candidate", value): + raise ValueError(reason) + return value diff --git a/server/router/directory.py b/server/router/directory.py index 235c97fa66..5d251f3551 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -782,29 +782,6 @@ def attach_refusal( return "" -def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> str: - """Why a posted fingerprint is not this row's cell ('' when it is): its stores and - hosts, the servers' own values, equal the row's.""" - fields = value["fields"] - row = found.row - expected = { - "deployment_id": row.deployment_id, - "bucket": found.bucket, - "bucket_prefix": found.bucket_prefix, - "public_host": row.host, - } - differ = [k for k, want in expected.items() if fields.get(k, cell_state.DEFAULTS[k]) != want] - if differ: - return f"{name}'s {', '.join(differ)} differ from cell {row.cell!r}'s registration" - issuer_host = fields.get("oauth_issuer_host", cell_state.DEFAULTS["oauth_issuer_host"]) - if issuer_host != row.host: - return ( - f"{name}'s oauth_issuer_host is not cell {row.cell!r}'s host {row.host!r}: the " - "router attaches no issuer host other than the cell's own" - ) - return "" - - def _as_json(*values: Any) -> str: """Values in one canonical JSON text, which tells `false`, `0` and `0.0` apart.""" return json.dumps(values, sort_keys=True) @@ -816,12 +793,19 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: if reason := attach_refusal(found, begin.deployment_id, begun=False): return reason row = found.row + registered = { + "cell": row.cell, + "deployment_id": row.deployment_id, + "bucket": found.bucket, + "bucket_prefix": found.bucket_prefix, + "host": row.host, + } for name, value in (("current_fp", begin.current_fp), ("next_fp", begin.next_fp)): if value is None: continue - if reason := _fingerprint_refusal(found, name, value) or fingerprints.placement_refusal( - row.cell, name, value - ): + if reason := fingerprints.registration_refusal(value, registered, name): + return reason + if reason := fingerprints.placement_refusal(row.cell, name, value): return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun # writes the fingerprints again only when they are the row's to the type. diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py index 5f8778b3a6..8b30894443 100644 --- a/server/router/fingerprints.py +++ b/server/router/fingerprints.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Any from server.core import cell, org_patterns @@ -56,6 +56,32 @@ def readable(value: Any) -> bool: return True +def registration_refusal(value: Any, registered: Mapping[str, Any], name: str = "candidate") -> str: + """A fingerprint accepts only the registered stores and host.""" + if not readable(value): + return f"{name}: unreadable fingerprint; change fingerprints with `loom cell update`" + expected = { + "deployment_id": registered["deployment_id"], + "bucket": registered["bucket"], + "bucket_prefix": registered["bucket_prefix"], + "public_host": registered["host"], + } + fields = value["fields"] + changed = [ + name for name, want in expected.items() if fields.get(name, cell.DEFAULTS[name]) != want + ] + if changed: + return ( + f"{name}'s {', '.join(changed)} differ from cell {registered['cell']!r}'s registration" + ) + if fields.get("oauth_issuer_host", cell.DEFAULTS["oauth_issuer_host"]) != registered["host"]: + return ( + f"{name}'s oauth_issuer_host differs from the registered host: the router " + "attaches no issuer host other than the cell's own" + ) + return "" + + def _pair_refusal(a: dict[str, Any], b: dict[str, Any]) -> str: if set(a.get("bearer_issuers", [])) & set(b.get("bearer_issuers", [])): return "shared bearer issuer reservation" diff --git a/tests/unit/test_cell_update_target.py b/tests/unit/test_cell_update_target.py new file mode 100644 index 0000000000..aefe6ceb1d --- /dev/null +++ b/tests/unit/test_cell_update_target.py @@ -0,0 +1,122 @@ +"""An update validates its fingerprint before reserving or accepting it.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell +from server.settings import Settings, settings + +REGISTERED = { + "cell": "first", + "host": "loom.example", + "deployment_id": "deployment", + "bucket": "docs", + "bucket_prefix": "originals", +} + + +def _config(**overrides: Any) -> Settings: + values = settings.model_dump() | { + "public_base_url": "https://loom.example", + "oauth_issuer": "https://loom.example", + "document_s3_bucket": "docs", + "document_s3_prefix": "originals", + "code_repos_json": "[]", + "code_org": "", + "tracker_org": "", + "oidc_jit_org": "", + "self_signup": "off", + "scim_token": "", + } + return Settings(_env_file=None, **(values | overrides)) + + +def test_target_uses_explicit_settings_and_the_existing_code_reader() -> None: + from server.cli.cell_update_config import target_fingerprint + + config = _config(code_org="configured", tracker_org="tracker") + before_config, before_registration = config.model_dump(), deepcopy(REGISTERED) + value = target_fingerprint(config, "deployment", REGISTERED) + assert value == cell.as_value(cell.fingerprint_for(config, "deployment", {"configured"})) + assert value["fields"]["configured_orgs"] == ["configured", "tracker"] + assert config.model_dump() == before_config + assert REGISTERED == before_registration + + +@pytest.mark.parametrize( + "change,field", + [ + ({"document_s3_bucket": "elsewhere"}, "bucket"), + ({"document_s3_prefix": "elsewhere"}, "bucket_prefix"), + ({"public_base_url": "https://elsewhere.example"}, "public_host"), + ({"oauth_issuer": "https://elsewhere.example"}, "oauth_issuer_host"), + ], +) +def test_target_refuses_changed_registered_identity_without_echoing_values( + change: dict[str, str], field: str +) -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match=field) as error: + target_fingerprint(_config(**change), "deployment", REGISTERED) + assert "elsewhere" not in str(error.value) + + +def test_target_refuses_different_deployment_and_bucketless_adoption() -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match="deployment_id"): + target_fingerprint(_config(), "different", REGISTERED) + with pytest.raises(ValueError, match="bucket"): + target_fingerprint( + _config(), "deployment", REGISTERED | {"bucket": "", "bucket_prefix": ""} + ) + + +def test_target_refuses_scim_before_any_state_work() -> None: + from server.cli.cell_update_config import target_fingerprint + + with pytest.raises(ValueError, match="per-cell SCIM") as error: + target_fingerprint(_config(scim_token="private-scim-token"), "deployment", REGISTERED) + assert "private-scim-token" not in str(error.value) + + +@pytest.mark.parametrize("org", ["-", "*", "x" * (cell.ORG_MAX_BYTES + 1)]) +def test_target_refuses_reserved_or_oversized_configured_orgs(org: str) -> None: + from server.cli.cell_update_config import target_fingerprint + + # A fully validated Settings model may still name a non-tenant configured org. + with pytest.raises(ValueError, match="fingerprint"): + target_fingerprint(_config(tracker_org=org), "deployment", REGISTERED) + + +def test_target_must_fit_a_live_placement() -> None: + from server.cli.cell_update_config import target_fingerprint + + config = _config( + code_repos_json=json.dumps( + [ + {"namespace": f"repo{n}", "org": str(n) + "x" * 1800, "repo": "owner/project"} + for n in range(35) + ] + ) + ) + with pytest.raises(ValueError, match="placement body"): + target_fingerprint(config, "deployment", REGISTERED) + + +def test_registration_check_applies_defaults_and_preserves_newer_fields() -> None: + from server.router.fingerprints import registration_refusal + + value = cell.as_value({"future": {"nested": [False, 0]}}) + before = deepcopy(value) + empty = REGISTERED | {"host": "", "deployment_id": "", "bucket": "", "bucket_prefix": ""} + assert registration_refusal(value, empty) == "" + assert value == before + assert "deployment_id" in registration_refusal(value, REGISTERED) + assert "unreadable" in registration_refusal({"v": True, "fields": {}}, empty) From 459dc34e49e94b19d5023472efe72bee27b3e8ad Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:50 +0000 Subject: [PATCH 059/181] Fix control batch bounds and schema coexistence checks Signed-off-by: zlareb1 --- docs/architecture/cells.md | 44 +++--- docs/operations/cell-attach.md | 4 +- server/cli/cell_attach_pg.py | 5 +- server/cli/cell_control.py | 2 +- server/router/app.py | 5 +- .../fixtures/router_placement_n_minus_one.py | 141 ++++++++++++++++++ tests/integration/test_cell_attach.py | 4 +- .../integration/test_cell_control_cleanup.py | 119 +++++++++++++++ tests/integration/test_pg_online_index.py | 37 +++-- .../test_pg_online_index_durability.py | 6 +- tests/unit/test_cell_attach.py | 18 +-- tests/unit/test_cell_control.py | 21 +++ tests/unit/test_cell_control_census.py | 36 ++++- tests/unit/test_router_attach.py | 14 ++ 14 files changed, 401 insertions(+), 55 deletions(-) create mode 100644 tests/integration/fixtures/router_placement_n_minus_one.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index a0079ac900..1954a69930 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -677,13 +677,13 @@ inserts cannot hold the write off, a new-org request waits on attach for a few s at most, and an attach that stalls holding the write's lock releases it within seconds. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with -server-side timeouts, then writes the mode and commits; after a failed try it ends the -mode transaction still in flight, found by its `ExclusiveLock` on `loom.tenancy_mode` in -this database (a pid captured at connect is wrong behind a connection path that swaps -backends), and waits until the lock clears. Whether the commit landed is decided by a -durable read of the mode, whose commit waits for the standby, since a commit ended while -it waited is on the primary alone ([attaching a cell](#attaching-a-cell): step 3 and the -durable read). +server-side timeouts, then writes the mode and commits on its exact ATTACH backend. +Only recognized pre-COMMIT contention can retry, after acknowledged rollback and a check +that this backend still holds ATTACH. Unexpected failure or an uncertain COMMIT closes +the owned session and ends the run. Attach neither finds nor ends another backend. A +fresh run takes the lock and claims again; whether an earlier commit landed is decided +by a durable mode read whose COMMIT waits for the standby +([attaching a cell](#attaching-a-cell): step 3 and the durable read). The mode cache reads the mode with a plain `SELECT`, which takes no row lock and never waits on attach. An insert that finds `enforced` (a process whose cache still reads `recording`) writes nothing: the request goes on to the placement call as in @@ -1065,7 +1065,7 @@ Then, in order (the step numbers are the ones the rest of this page uses): checks and the standbys again, and the scan's elapsed time within half of `--scan-budget`, since step 4 reads the same tables again. Each refuses in `recording`, which can still be backed out. -- **Step 3, the mode commit**: `attach/enforced`, then on a session of its own an EXCLUSIVE +- **Step 3, the mode commit**: `attach/enforced`, then on the same ATTACH session an EXCLUSIVE lock on `loom.tenancy_mode`, 2 s a try within `--wait`, the attach lock confirmed and the cheap heartbeat refusals run before each try (a booting process skips the try). Each try, including COMMIT, has a client deadline: the lock wait plus a 3 s statement margin and @@ -1077,10 +1077,11 @@ Then, in order (the step numbers are the ones the rest of this page uses): failure or exhausted cleanup time aborts the transport and preserves the original failure; abort alone does not prove that server locks disappeared. A commit past its deadline has an unknown outcome until the durable read. - After a failed try, the mode transaction an earlier run left in flight is found by its lock (only - attach takes `ExclusiveLock` there; a relation's OID is matched in this database alone) - and ended. From the commit on the mode only rises: every later refusal leaves the cell - `enforced` and `attaching`, and a rerun resumes from the durable read. + Only acknowledged pre-COMMIT rollback with ATTACH still held permits another try on + that backend. Any other failure ends the run without terminating another backend; + rerun to take the lock and claim afresh. From an acknowledged commit on, the mode + only rises: every later refusal leaves the cell `enforced` and `attaching`, and a + rerun resumes from the durable read. - **The durable read**, in every run that reads `enforced`: the mode and the `first` row read in a transaction that emits a transactional logical message, whose commit waits for the synchronous standbys, with the standbys checked attached just before and after. Then @@ -1312,17 +1313,18 @@ participate in this protocol. - `loom cell limit --cell --closed` closes a cell to new orgs, `--open` reopens a serving cell, and `--placements-max ` sets its placement quota (`--placements-max none` clears it). -- `loom cell remove --cell ` removes the directory row of a cell that is `attaching` - with no placement (`--cell` is the cell's `LOOM_CELL`, unset by then). For the first cell - it reads the cell's Postgres alone, on one session from `LOOM_PG_DSN` with no ClickHouse, - and refuses unless it reads `open` and records the row's deployment id, which enforces the - back-out order: unset `LOOM_CELL` on every process (each serves as in `recording`), then - the full-state cleanup, then the removal +- `loom cell remove --cell ` removes the directory row of an `attaching` cell + with no placement. For both first and later cells, keep `LOOM_CELL=` and the + registered store settings on the control client. Removal takes the direct marked + ATTACH backend, verifies the paired deployment and registration, and claims a fresh + directory generation before deleting the row. The first cell must also read `open`, + enforcing the back-out order: stop or unset `LOOM_CELL` on the accepting processes, + complete full-state cleanup, then remove the directory row ([troubleshooting](../operations/troubleshooting.md#a-process-with-loom_cell-unset-is-gated)). Removing the row first answers every new org 503; clearing the state first gates every - process with `LOOM_CELL` set. A first cell with a placement goes forward, never back. A - removed cell's stores keep whatever state rows they hold, so the full-state cleanup runs - before they are prepared again. + accepting process with `LOOM_CELL` set. A first cell with a placement goes forward, + never back. Removal leaves the stores' state rows intact; full-state cleanup must + precede preparing those stores again. - The router's operator credential is `lco_` and 128 lowercase hex (a router with no `LOOM_ROUTER_OPERATOR_SHA256` answers the operator API 503 with `NO_OPERATOR_CREDENTIAL`), which `loom cell credential new --operator` makes; the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 13c65405f6..20b541aec3 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -55,8 +55,8 @@ Every item below is a precondition attach checks, or one it cannot check and rel `max_prepared_transactions = 0`; the database encoding is UTF8. 13. No org longer than 2048 bytes is in the cell's stores (below). `--dry-run` lists any. -Attach's Postgres role needs the privileges of `pg_read_all_stats` and `pg_signal_backend` -without `SET ROLE` (grant their memberships with `INHERIT TRUE`), `USAGE` on schema +Attach's Postgres role needs the privileges of `pg_read_all_stats` +without `SET ROLE` (grant its membership with `INHERIT TRUE`), `USAGE` on schema `loom`, `SELECT` on `loom.cell_first`, `loom.tenancy_mode`, `loom.cell_self`, `loom.authority` and every org table, `UPDATE` on `loom.tenancy_mode`, `INSERT` on `loom.tenancy`, `SELECT`, `INSERT`, `UPDATE` and `DELETE` on `loom.cell_probe`, `TEMPORARY` diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index d18ff6f628..abf23bbf43 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -399,8 +399,6 @@ async def session_passes() -> bool: SELECT need, held FROM (VALUES ('effective privileges of pg_read_all_stats', pg_has_role(current_user, 'pg_read_all_stats', 'USAGE')), - ('effective privileges of pg_signal_backend', - pg_has_role(current_user, 'pg_signal_backend', 'USAGE')), ('USAGE on schema loom', has_schema_privilege('loom', 'USAGE')), ('SELECT on loom.cell_first', has_table_privilege('loom.cell_first', 'SELECT')), ('SELECT on loom.tenancy_mode', has_table_privilege('loom.tenancy_mode', 'SELECT')), @@ -582,8 +580,7 @@ def _busy_refusal(table: str, holders: list[Any]) -> Refusal: # -- step 3: the mode commit ------------------------------------------------------------ -# The mode transaction in flight: the backend holding or waiting for the ExclusiveLock on -# loom.tenancy_mode in this database, which only attach takes. +# Only acknowledged pre-COMMIT contention permits another try on the same session. _TIMED_OUT = frozenset({"55P03", "57014"}) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 7fd92e28ab..c4360839ed 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -138,7 +138,7 @@ async def _open(self, what: str) -> None: self.conn = await pg._connect_dedicated(statement_timeout_ms=60_000) if self.closed: # A connector returning after cancellation cannot publish a usable session. - await pg.close_owned(self.conn, end=self.close_end or 0) + await self.close() raise _refused("connection arrived after the control attempt ended") try: await pg._check_deployment(self.conn) diff --git a/server/router/app.py b/server/router/app.py index 39a1ac23c8..52c9d3d157 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -723,7 +723,10 @@ async def attach_place(self, request: Request) -> JSONResponse: expected = _control(body) if isinstance(expected, JSONResponse): return expected - if len(json.dumps(body, ensure_ascii=True).encode()) > MAX_BODY_BYTES: + if ( + len(json.dumps(body, ensure_ascii=True, separators=(",", ":")).encode()) + > MAX_BODY_BYTES + ): return _refused( 413, f"a placement batch without its control envelope is at most {MAX_BODY_BYTES} bytes", diff --git a/tests/integration/fixtures/router_placement_n_minus_one.py b/tests/integration/fixtures/router_placement_n_minus_one.py new file mode 100644 index 0000000000..dea9c92c22 --- /dev/null +++ b/tests/integration/fixtures/router_placement_n_minus_one.py @@ -0,0 +1,141 @@ +"""Frozen pre-generation router placement/query code for schema coexistence tests. + +The serving statements and row model are copied from the checkpoint before the +control generation column. Connection acquisition/error handling are injected from +the test directory; no current placement projection or writer is imported. +""" + +from __future__ import annotations + +import hmac +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from server.core import cell as cell_state +from server.router.directory import OrgTooLong, PlacementRefused, _connection, _reading + +_PLACE_LOCK = 0x6C726F70 # "lrop" + + +@dataclass(frozen=True) +class Cell: + """What the router's APIs read of a cell's row.""" + + cell: str + is_first: bool + host: str + state: str + mode: str + closed: bool + placements_max: int | None + placed: int + credential_sha256: str + deployment_id: str + current_fp: dict[str, Any] | None + next_fp: dict[str, Any] | None + # The new credential's hash during a rotation (`loom cell credential --next`). + credential_next_sha256: str | None + + def holds(self, presented: str) -> bool: + """Whether `presented` (a credential's SHA-256) is this cell's, current or next.""" + hashes = (self.credential_sha256, self.credential_next_sha256) + return holds(presented, [h for h in hashes if h]) + + +def holds(presented: str, hashes: Iterable[str]) -> bool: + """Whether `presented` (a credential's SHA-256) is one of `hashes` (each set), in + constant time each.""" + return any(hmac.compare_digest(presented, h) for h in hashes) + + +_CELL_COLUMNS = ( + "cell, is_first, host, state, mode, closed, placements_max, placed, credential_sha256, " + "credential_next_sha256, deployment_id, current_fp, next_fp" +) + +_CELL = f"SELECT {_CELL_COLUMNS} FROM loom_router.cells" + +_WINNER = ( + "SELECT p.cell, c.deployment_id FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1" +) + + +def _cell(row: Any) -> Cell: + return Cell(**dict(row)) + + +async def place(org: str, cell_name: str, refusal: Callable[[Cell], str]) -> tuple[str, str]: + """Place `org` in `cell_name` unless it is placed already; the (cell, deployment id) + that holds it. An org placed already answers its cell, whatever the cell's state. + Otherwise the cell's row is locked, the winner read again (an attach batch holds the row + lock, not the org's), and the row judged (`refusal`, then its quota), so a change to it + waits for the placement and the quota holds across router replicas; a refused placement + raises PlacementRefused and writes nothing, and a new org name longer than + `cell.ORG_MAX_BYTES` raises OrgTooLong (one placed before the limit answers its cell, as + any placed org does).""" + async with _reading(), _connection() as conn, conn.transaction(): + # Placements of one org run one at a time, whichever cell asks: the later one + # reads the earlier one's winner. + await conn.execute("SELECT pg_advisory_xact_lock($1, hashtext($2))", _PLACE_LOCK, org) + row = await conn.fetchrow(_WINNER, org) + if row is None: + _refuse_long([org]) + locked = await conn.fetchrow(f"{_CELL} WHERE cell = $1 FOR UPDATE", cell_name) + if locked is None: + raise PlacementRefused(f"cell {cell_name!r} is not in the directory") + # An attach batch of this cell, which holds the row lock and not the org's, may + # have placed the org since the first read: it is answered as placed. + row = await conn.fetchrow(_WINNER, org) + if row is None: + await _place_one(conn, org, cell_name, _cell(locked), refusal) + row = await conn.fetchrow(_WINNER, org) + return str(row["cell"]), str(row["deployment_id"]) + + +async def _place_one( + conn: Any, org: str, cell_name: str, candidate: Cell, refusal: Callable[[Cell], str] +) -> None: + """Judge the locked row (`refusal`, then its quota) and insert the placement; its count + and audit row only when the insert wrote it.""" + if reason := refusal(candidate): + raise PlacementRefused(reason) + quota = candidate.placements_max + if quota is not None and candidate.placed >= quota: + raise PlacementRefused(f"cell {cell_name!r} holds its quota of orgs") + inserted = await conn.fetchval( + "INSERT INTO loom_router.placements (org, cell) VALUES ($1, $2) " + "ON CONFLICT DO NOTHING RETURNING org", + org, + cell_name, + ) + if inserted is None: + return + await conn.execute( + "UPDATE loom_router.cells SET placed = placed + 1 WHERE cell = $1", cell_name + ) + await conn.execute( + "INSERT INTO loom_router.audit (actor, action, detail) VALUES ($1, 'place', $2)", + f"cell:{cell_name}", + {"org": org, "cell": cell_name}, + ) + + +def _refuse_long(orgs: Iterable[str]) -> None: + """OrgTooLong for the first org longer than `cell.ORG_MAX_BYTES`: the one check every + route that inserts a placement runs, so no route places such an org.""" + for org in orgs: + if reason := cell_state.org_name_refusal(org): + raise OrgTooLong(reason) + + +async def lookup(org: str) -> tuple[str, str] | None: + """The (cell, host) an org is placed in, or None.""" + async with _reading(), _connection(lookup=True) as conn: + row = await conn.fetchrow( + "SELECT c.cell, c.host FROM loom_router.placements AS p " + "JOIN loom_router.cells AS c USING (cell) WHERE p.org = $1", + org, + ) + return (str(row["cell"]), str(row["host"])) if row else None diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 6cbc595bd5..a84474806e 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -809,7 +809,7 @@ async def test_signaling_another_role_requires_an_effective_signal_grant(cell_st @pytest.mark.asyncio -@pytest.mark.parametrize("capability", ["pg_read_all_stats", "pg_signal_backend"]) +@pytest.mark.parametrize("capability", ["pg_read_all_stats"]) async def test_attach_needs_its_predefined_capabilities_without_set_role( cell_stores, monkeypatch, capsys, capability: str ) -> None: @@ -829,7 +829,7 @@ async def as_attacher(**kwargs: Any) -> Any: async with real_dedicated() as conn: owner = await conn.fetchval("SELECT current_user") await conn.execute(f"GRANT {owner} TO {role} WITH INHERIT TRUE, SET FALSE") - await conn.execute(f"GRANT pg_read_all_stats, pg_signal_backend TO {role}") + await conn.execute(f"GRANT pg_read_all_stats TO {role}") await conn.execute(f"GRANT {capability} TO {role} WITH INHERIT FALSE") assert await conn.fetchval("SELECT pg_has_role($1, $2, 'MEMBER')", role, capability) assert not await conn.fetchval("SELECT pg_has_role($1, $2, 'USAGE')", role, capability) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index f66eeb5169..e057fef323 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -293,3 +293,122 @@ async def read(self): with pytest.raises(asyncio.CancelledError, match="native attach cancellation"): await asyncio.wait_for(task, 1.3) await released(native_database, backend["pid"]) + + +async def test_attach_role_needs_stats_but_not_backend_signal_privilege(native_database): + import uuid + + from server.cli import cell_attach_pg as apg + + role = f"it_attach_grants_{uuid.uuid4().hex[:10]}" + conn = native_database + database = await conn.fetchval("SELECT current_database()") + await conn.execute(f"CREATE ROLE {role}") + try: + await conn.execute(f"GRANT USAGE ON SCHEMA loom TO {role}") + await conn.execute( + f"GRANT SELECT ON loom.cell_first, loom.tenancy_mode, loom.cell_self, loom.authority TO {role}" + ) + await conn.execute(f"GRANT UPDATE ON loom.tenancy_mode TO {role}") + await conn.execute(f"GRANT INSERT ON loom.tenancy TO {role}") + await conn.execute(f"GRANT SELECT, INSERT, UPDATE, DELETE ON loom.cell_probe TO {role}") + await conn.execute(f'GRANT TEMPORARY ON DATABASE "{database}" TO {role}') + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT FALSE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + refused = await apg.own_role_refusals(conn, []) + assert [r.reason for r in refused] == [ + "attach's role lacks effective privileges of pg_read_all_stats" + ] + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"GRANT pg_read_all_stats TO {role} WITH INHERIT TRUE") + await conn.execute(f"SET SESSION AUTHORIZATION {role}") + assert not await conn.fetchval( + "SELECT pg_has_role(current_user, 'pg_signal_backend', 'USAGE')" + ) + assert await apg.own_role_refusals(conn, []) == [] + finally: + await conn.execute("RESET SESSION AUTHORIZATION") + await conn.execute(f"DROP OWNED BY {role}") + await conn.execute(f"DROP ROLE {role}") + + +@pytest.mark.parametrize("late", [False, True]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_native_public_close_fatal_stays_inside_owner(native_database, late, kind): + import os + import sys + + from tests.integration._pg_database import with_database + + dsn = with_database( + os.environ["LOOM_PG_DSN"], await native_database.fetchval("SELECT current_database()") + ) + script = r""" +import asyncio, os, sys +import asyncpg +from server.cli import cell_control +from server.core import pg +from server.settings import settings +late, kind = sys.argv[1:] +settings.pg_dsn = os.environ['LOOM_NATIVE_CLEANUP_DSN'] +settings.cell = 'first' +pg._expected = None +original = SystemExit(17) if kind == 'exit' else KeyboardInterrupt('close interrupted') +real_connect, real_close = pg._connect_dedicated, asyncpg.Connection.close +backend = {} +async def connect(**kwargs): + conn = await real_connect(**kwargs) + backend['pid'] = conn.get_server_pid() + if late == 'True': + try: await asyncio.Future() + except asyncio.CancelledError: pass + return conn +async def close(self, **kwargs): + ours = self.get_server_pid() == backend.get('pid') + await real_close(self, **kwargs) + if ours: + backend['closes'] = backend.get('closes', 0) + 1 + raise original +async def body(owner): + await owner.conn.execute('SELECT * FROM probe FOR UPDATE') + return 19 +pg._connect_dedicated = connect +asyncpg.Connection.close = close +if late == 'True': cell_control.OPEN_S = .5 +async def main(): + observer = await asyncpg.connect(settings.pg_dsn) + try: + try: await cell_control.run('native close', body) + except BaseException as caught: + if late == 'True': assert isinstance(caught, TimeoutError), type(caught) + else: assert caught is original, type(caught) + assert backend['closes'] == 1 + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)', backend['pid']) + assert not await observer.fetchval('SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)', backend['pid']) + print('LOOP_STILL_RUNNING') + else: raise AssertionError('required close fatal became success') + finally: + asyncpg.Connection.close = real_close + await observer.close(timeout=1) +asyncio.run(main()) +""" + child = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + str(late), + kind, + env={**os.environ, "LOOM_NATIVE_CLEANUP_DSN": dsn}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(child.communicate(), 10) + assert child.returncode == 0 and b"LOOP_STILL_RUNNING" in stdout, stderr.decode() + finally: + if child.returncode is None: + child.kill() + await child.wait() diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index 1995e73bee..38638ad877 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -55,6 +55,14 @@ def _prefix(tmp_path: Path) -> Path: return prefix +def _journal() -> list[str]: + return sorted(path.name for path in ROUTER_MIGRATIONS_DIR.glob("*.sql")) + + +def _after_prefix() -> list[str]: + return _journal()[3:] + + async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[str]: return await pg_migrate.migrate(files, connect=connect, schema="loom_router", lock_key=8917423) @@ -62,7 +70,7 @@ async def _migrate(connect: Any, files: Path = ROUTER_MIGRATIONS_DIR) -> list[st async def test_online_migration_installs_and_retries_from_both_schemas(database, tmp_path) -> None: conn, connect = database await _migrate(connect, _prefix(tmp_path)) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await _migrate(connect) == [] index = pg_online_index.parse( (ROUTER_MIGRATIONS_DIR / "004_placements_by_cell_org.sql").read_text() @@ -73,7 +81,7 @@ async def test_online_migration_installs_and_retries_from_both_schemas(database, async def test_install_from_empty(database) -> None: _, connect = database - assert len(await _migrate(connect)) == 4 + assert await _migrate(connect) == _journal() async def _until(conn: Any, sql: str, *args: Any) -> None: @@ -132,12 +140,19 @@ async def test_old_snapshot_old_blocking_migrator_and_two_new_migrators(database "(cell, host, state, credential_sha256, deployment_id) " "VALUES ('a', 'a.example.test', 'serving', repeat('a', 64), 'dep-a')" ) - assert await directory.place("org-a", "a", lambda _: "") == ("a", "dep-a") - assert await directory.lookup("org-a") == ("a", "a.example.test") + from tests.integration.fixtures import router_placement_n_minus_one as serving + + assert await serving.place("org-a", "a", lambda _: "") == ("a", "dep-a") + assert await serving.lookup("org-a") == ("a", "a.example.test") + assert await conn.fetchval("SELECT placed FROM loom_router.cells WHERE cell = 'a'") == 1 + assert ( + await conn.fetchval("SELECT count(*) FROM loom_router.audit WHERE action = 'place'") + == 1 + ) await tx.rollback() async with asyncio.timeout(5): results = await asyncio.gather(first, second) - assert sorted(results) == [[], ["004_placements_by_cell_org.sql"]] + assert sorted(results) == [[], _after_prefix()] finally: if snapshot.is_in_transaction(): await tx.rollback() @@ -182,7 +197,7 @@ async def test_cancelled_build_leaves_a_matching_invalid_index_and_retry_repairs if not migrating.done(): migrating.cancel() await asyncio.gather(migrating, return_exceptions=True) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await _migrate(connect) == [] @@ -241,7 +256,7 @@ async def completed_then_cancel(*args, **kwargs): await _migrate(connect) oid = await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") monkeypatch.setattr(pg_online_index, "build", original) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() assert await conn.fetchval("SELECT 'loom_router.placements_by_cell_org'::regclass::oid") == oid @@ -347,7 +362,7 @@ async def direct_drop(): if not dropping.done(): dropping.cancel() await asyncio.gather(dropping, return_exceptions=True) - assert await _migrate(connect) == ["004_placements_by_cell_org.sql"] + assert await _migrate(connect) == _after_prefix() async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) -> None: @@ -364,7 +379,7 @@ async def test_try_lock_wait_has_no_retained_transaction_or_snapshot(database) - ) assert not migrating.done() await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") - assert len(await asyncio.wait_for(migrating, 5)) == 4 + assert await asyncio.wait_for(migrating, 5) == _journal() finally: await conn.execute("SELECT pg_advisory_unlock(8917423::bigint)") if not migrating.done(): @@ -376,6 +391,8 @@ async def test_online_validation_is_repeated_after_later_files(database, tmp_pat _, connect = database for source in ROUTER_MIGRATIONS_DIR.glob("*.sql"): (tmp_path / source.name).write_bytes(source.read_bytes()) - (tmp_path / "005_damage.sql").write_text("DROP INDEX loom_router.placements_by_cell_org;") + ( + tmp_path / f"{max(int(name.split('_')[0]) for name in _journal()) + 1:03d}_damage.sql" + ).write_text("DROP INDEX loom_router.placements_by_cell_org;") with pytest.raises(ValueError, match="recorded online-index"): await _migrate(connect, tmp_path) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py index e5644af5af..f38453e747 100644 --- a/tests/integration/test_pg_online_index_durability.py +++ b/tests/integration/test_pg_online_index_durability.py @@ -158,7 +158,11 @@ async def pause_before_ledger(*args, **kwargs): await _waiting(observer, retry) await _docker("unpause", STANDBY) paused = False - assert await asyncio.wait_for(retry, 8) == [] + assert await asyncio.wait_for(retry, 8) == sorted( + path.name + for path in ROUTER_MIGRATIONS_DIR.glob("*.sql") + if path.name > "004_placements_by_cell_org.sql" + ) observer.terminate() await _docker("kill", "--signal", "KILL", PRIMARY) await _docker( diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index a69c5c5c8f..5188641547 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -239,7 +239,6 @@ def lock(sql: str, *_a: Any) -> str: conn = FakeConn( [ ("SET LOCAL lock_timeout", lambda *a: lock(conn.executed[-1])), - ("l.mode = 'ExclusiveLock'", []), ] ) @@ -487,6 +486,7 @@ async def fetchval(self, sql: str, *args: Any) -> Any: return self._answer(sql, args) async def execute(self, sql: str, *args: Any) -> str: + assert "pg_terminate_backend" not in sql and "pg_cancel_backend" not in sql self.executed.append(sql) for needle, answer in self.routes: if needle in sql: @@ -697,16 +697,10 @@ def lock(*_a: Any) -> str: raise answer return "LOCK TABLE" - holders = _seq( - [{"pid": 5, "application_name": "attach", "xact_start": "t", "rolsuper": False}], [] - ) conn = FakeConn( [ ("LOCK TABLE", lock), ("UPDATE loom.tenancy_mode", "UPDATE 1"), - ("l.mode = 'ExclusiveLock'", holders), - ("rolsuper FROM pg_roles", False), - ("pg_terminate_backend", True), ] ) confirmed: list[bool] = [] @@ -733,6 +727,11 @@ async def ready() -> list[str]: assert len(confirmed) == 2 assert conn.executed.count("ROLLBACK") == 1 assert "TERMINATE" not in conn.executed + assert not any( + "pg_terminate_backend" in sql or "pg_cancel_backend" in sql for sql in conn.executed + ) + assert conn.executed.count("COMMIT") == 1 + assert sum(sql.startswith("UPDATE loom.tenancy_mode") for sql in conn.executed) == 1 @pytest.mark.asyncio @@ -742,7 +741,7 @@ async def test_the_mode_commit_refuses_past_wait_and_skips_a_try_while_booting( monkeypatch: pytest.MonkeyPatch, wait_s: int, ) -> None: - conn = FakeConn([("l.mode = 'ExclusiveLock'", [])]) + conn = FakeConn() run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] run.owner = _owner(FakeConn()) run.own_fp = (CURRENT, NEXT) @@ -1219,7 +1218,6 @@ async def test_wait_0_runs_one_try_of_each_wait( [ ("LOCK TABLE", "LOCK TABLE"), ("UPDATE loom.tenancy_mode", "UPDATE 1"), - ("l.mode = 'ExclusiveLock'", []), ] ) marks: list[str] = [] @@ -1282,7 +1280,7 @@ async def test_a_mode_commit_refused_without_a_landed_try_says_recording( def lock(*_a: Any) -> str: raise _Timeout() - conn = FakeConn([("LOCK TABLE", lock), ("l.mode = 'ExclusiveLock'", [])]) + conn = FakeConn([("LOCK TABLE", lock)]) async def call(_self: Any, _path: str, _body: Any = None) -> dict[str, Any]: return {"cell": "first"} diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 6b2b821b91..ccbbb7b6f9 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -284,3 +284,24 @@ async def opening(self, _what): with pytest.raises(OSError, match="required close failed"): await cell_control.run("test", AsyncMock(return_value=7)) assert "TERMINATE" in conn.executed + + +@pytest.mark.asyncio +async def test_late_connection_arriving_inside_reserve_is_closed_once(monkeypatch): + from server.core import pg + + conn = FakeConn() + + async def connect(**_): + try: + await asyncio.Future() + except asyncio.CancelledError: + return conn + + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(cell_control, "OPEN_S", 0.08) + work = AsyncMock() + with pytest.raises(TimeoutError): + await cell_control.run("late", work) + work.assert_not_awaited() + assert conn.executed == ["CLOSE"] diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 0c358f2c38..45b8e85baa 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -19,16 +19,16 @@ TABLES = r"(?:loom\.(?:cell_self|cell_first|tenancy_mode)|loom_router\.[A-Za-z_]+)" TARGET = rf"(?:{TABLES}|loom(?:_router)?\.\{{[^}}]+\}}|\{{[^}}]+\}})" WRITES = re.compile( - rf"(?:^|;)\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" + rf"(?:^|;|[()])\s*(INSERT\s+INTO|UPDATE|DELETE\s+FROM|TRUNCATE(?:\s+TABLE)?|" rf"ALTER\s+TABLE(?:\s+IF\s+EXISTS)?|CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?|" rf"DROP\s+TABLE(?:\s+IF\s+EXISTS)?)\s+({TARGET})" - rf"|(?:^|;)\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", + rf"|(?:^|;|[()])\s*((?:CREATE|DROP)\s+TRIGGER)\b[^;]*?\bON\s+({TARGET})", re.IGNORECASE, ) def mutations(sql: str) -> list[tuple[str, str]]: - sql = re.sub(r"--[^\n]*", "", sql).replace('"', "") + sql = re.sub(r"--[^\n]*|/\*.*?\*/", "", sql, flags=re.DOTALL).replace('"', "") return [(" ".join((a or c).upper().split()), b or d) for a, b, c, d in WRITES.findall(sql)] @@ -505,6 +505,22 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): ("statement", "expected"), [ ('"UPDATE loom.cell_self SET current_fp = $1"', ("UPDATE", "loom.cell_self")), + ( + '"WITH expected AS (SELECT 1) UPDATE loom.cell_self SET current_fp = $1"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH expected AS (SELECT 1) DELETE FROM loom_router.cells WHERE cell = $1"', + ("DELETE FROM", "loom_router.cells"), + ), + ( + '"WITH changed AS (UPDATE loom.cell_self SET current_fp = $1 RETURNING 1) SELECT * FROM changed"', + ("UPDATE", "loom.cell_self"), + ), + ( + '"WITH outer_cte AS (WITH inner_cte AS (SELECT 1) UPDATE loom_router.cells SET closed = true RETURNING cell) SELECT * FROM outer_cte"', + ("UPDATE", "loom_router.cells"), + ), ( '"INSERT INTO loom_router.cells (cell) VALUES ($1)"', ("INSERT INTO", "loom_router.cells"), @@ -550,3 +566,17 @@ def test_acceptance_writer_callsites_are_classified(): ("server/cli/cell_attach.py", "Run.commit_mode", "apg.commit_enforced"): 1, } ) + + +def test_cte_data_modification_classifies_every_literal_sink(): + source = """ +async def unclassified(conn): + await conn.execute("WITH changed AS (DELETE FROM loom.cell_self RETURNING 1) " + "/* body done */ UPDATE loom_router.cells SET closed = true") +""" + assert inventory(source).writes == Counter( + { + ("unclassified", "DELETE FROM", "loom.cell_self"): 1, + ("unclassified", "UPDATE", "loom_router.cells"): 1, + } + ) diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index bffc8aefea..821272bcdf 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1437,3 +1437,17 @@ async def _run() -> None: assert again.json()["state"] == "serving" asyncio.run(_run()) + + +def test_native_compact_batch_with_control_envelope_fits_public_handler(client, store): + from server.cli.cell_attach import _encoded_size, batches + + _enforced(client) + orgs = [f"{i:03d}" + "x" * 127 for i in range(500)] + batch = batches(orgs, "dep-1")[0] + body = {"deployment_id": "dep-1", "orgs": batch} + assert _encoded_size(body) <= MAX_BODY_BYTES < len(json.dumps(body).encode()) + response = _call(client, "POST", "attach/place", body) + assert response.status_code == 200, response.text + assert response.json()["placed"] == len(batch) + assert set(store.placements) == set(batch) From 9caeb23ec169886d6cc028fc4330045591783677 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:05 +0000 Subject: [PATCH 060/181] Preserve native migration failures through owned cleanup Capture native failures inside owned work and public-close child tasks so their supervisors can clean up before rethrowing. Keep the first migration failure through context exit within its original cleanup deadline. Signed-off-by: zlareb1 --- server/core/pg.py | 14 ++- server/core/schema/pg_migrate.py | 75 +++++++++---- tests/unit/test_pg_close_owned.py | 75 +++++++++++++ tests/unit/test_pg_owned_fatal.py | 172 ++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_pg_owned_fatal.py diff --git a/server/core/pg.py b/server/core/pg.py index d7bdf5b3e4..e70c702582 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -392,16 +392,26 @@ async def close_owned(conn: Any, *, end: float) -> None: is a bounded fallback, not evidence that the backend has released its locks. The owner preserves an earlier operation failure when this cleanup fails. """ + + async def close() -> SystemExit | KeyboardInterrupt | None: + try: + await conn.close(timeout=max(0, end - time.monotonic())) + except (SystemExit, KeyboardInterrupt) as error: + # A native fatal in a child Task stops the loop before its owner can clean up. + return error + return None + task: asyncio.Task[Any] | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task = asyncio.create_task(conn.close(timeout=remaining)) + task = asyncio.create_task(close()) done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) if not done: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task.result() + if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): + raise result except BaseException: _terminate(conn) if task is not None: diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 371f61a449..edc5cdb927 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -21,7 +21,7 @@ import logging import time from asyncio import sleep -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager from pathlib import Path from typing import Any @@ -110,24 +110,61 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files, indexes = _validated(mig_dir, schema) - async with (connect or pg.dedicated)() as conn: - task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) - try: - return await asyncio.shield(task) - except BaseException: - end = time.monotonic() + pg._RELEASE_TIMEOUT_S - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. - pg._terminate(conn) - raise + cm = (connect or pg.dedicated)() + conn = await cm.__aenter__() + task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + try: + result = await asyncio.shield(task) + if isinstance(result, (SystemExit, KeyboardInterrupt)): + raise result + except BaseException as error: + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + await _failed_migration(cm, conn, task, error, end) + raise + await cm.__aexit__(None, None, None) + return result + + +async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: + try: + return await work + except (SystemExit, KeyboardInterrupt) as error: + # Transport native failures through child Tasks without stopping their supervisor. + return error + + +async def _failed_migration( + cm: AbstractAsyncContextManager[Any], + conn: Any, + task: asyncio.Task[Any], + error: BaseException, + end: float, +) -> None: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. + pg._terminate(conn) + release = asyncio.create_task( + _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + ) + try: + done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("migration context cleanup deadline expired") + release.result() + except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. + pg._terminate(conn) + finally: + if not release.done(): + release.cancel() + release.add_done_callback(pg._retrieve_task) def _validated( diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index fbbe778a5d..1ba7ab1f6a 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -142,3 +142,78 @@ async def fail(*_: Any) -> None: with pytest.raises(ValueError, match="migration failed"): await pg_migrate.migrate(tmp_path, connect=connect, schema="test") assert conn.aborted + + +async def test_successful_migration_still_reports_context_exit_failure( + monkeypatch, tmp_path +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + original = ValueError("context exit failed") + + @contextlib.asynccontextmanager + async def connect(): + yield Connection() + raise original + + async def applied(*_: Any) -> list[str]: + return ["001.sql"] + + monkeypatch.setattr(pg_migrate, "_migrate", applied) + with pytest.raises(ValueError) as error: + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert error.value is original + + +@pytest.mark.parametrize("cancel_work", [False, True]) +async def test_second_cancellation_during_migration_context_exit_keeps_first_failure( + monkeypatch, tmp_path, cancel_work +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + conn = Connection() + conn.finished.set() + started, exiting = asyncio.Event(), asyncio.Event() + observed: list[BaseException] = [] + original = ValueError("migration failed") + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + except BaseException as error: + observed.append(error) + exiting.set() + await asyncio.Future() + raise + + async def work(*_: Any) -> list[str]: + started.set() + if cancel_work: + await asyncio.Future() + raise original + + monkeypatch.setattr(pg_migrate, "_migrate", work) + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.1) + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await asyncio.wait_for(started.wait(), 1) + before = time.monotonic() + if cancel_work: + task.cancel("first cancellation") + await asyncio.wait_for(exiting.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(BaseException) as result: + await asyncio.wait_for(task, 0.3) + assert time.monotonic() - before < 0.3 + assert result.value is observed[0] + assert ( + isinstance(result.value, asyncio.CancelledError) + if cancel_work + else result.value is original + ) + assert len(conn.timeouts) == 1 + assert conn.aborted diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py new file mode 100644 index 0000000000..683dbca782 --- /dev/null +++ b/tests/unit/test_pg_owned_fatal.py @@ -0,0 +1,172 @@ +"""Owned child tasks preserve native failures until their supervisor cleans up.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize( + "phase", + ["close", "migration", "migration_exit", "migration_ordinary_exit", "migration_exit_stall"], +) +def test_native_failure_reaches_the_owner_after_cleanup(kind: str, phase: str, tmp_path) -> None: + program = textwrap.dedent( + """ + import asyncio + import contextlib + import sys + import time + from pathlib import Path + from server.core import pg + from server.core.schema import pg_migrate + + kind, phase, path = sys.argv[1:] + fatal_type = SystemExit if kind == 'SystemExit' else KeyboardInterrupt + original = ValueError('original') if 'ordinary' in phase else fatal_type('original') + secondary = fatal_type('cleanup') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': + raise original + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + finally: + events.append('exit') + if phase == 'migration_exit_stall': + await asyncio.Future() + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + + async def fail(*args): + events.append('work') + raise original + + async def main(): + pg_migrate._migrate = fail + pg._RELEASE_TIMEOUT_S = 0.05 + started = time.monotonic() + try: + if phase == 'close': + await pg.close_owned(conn, end=time.monotonic() + 1) + else: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert 'close' in events, events + if phase != 'close': + assert 'exit' in events, events + assert events.index('close') < events.index('exit'), events + else: + assert 'abort' in events, events + await asyncio.sleep(0) + print('owner-received-original-after-cleanup') + return + raise AssertionError('native failure was swallowed') + + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "owner-received-original-after-cleanup" in result.stdout + + +@pytest.mark.parametrize("connection_context", ["cell", "router"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt", "ValueError"]) +@pytest.mark.parametrize("exit_behavior", ["fatal", "stalled"]) +def test_actual_connection_context_keeps_first_failure_and_cleanup_end( + connection_context: str, kind: str, exit_behavior: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from pathlib import Path + import asyncpg + from server.core import pg + from server.core.schema import pg_migrate + from server.router import directory + + context, kind, exit_behavior, path = sys.argv[1:] + original = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt, + 'ValueError': ValueError}[kind]('original') + secondary = SystemExit('context failed') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if events.count('close') > 1: + if exit_behavior == 'stalled': + await asyncio.Future() + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + async def raw(*args, **kwargs): + return conn + async def checked(*args): + pass + async def fail(*args): + raise original + + async def main(): + pg.settings.pg_dsn = "postgresql://unused/test" + pg._check_deployment = checked + pg.init_connection = checked + asyncpg.connect = raw + directory._require_dsn = lambda: 'unused-test-dsn' + pg._RELEASE_TIMEOUT_S = 0.05 + pg_migrate._migrate = fail + connect = None if context == 'cell' else directory._dedicated + started = time.monotonic() + try: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert events.count('close') == 2, events + if exit_behavior == 'stalled': + assert 'abort' in events, events + await asyncio.sleep(0) + print('first-failure-and-original-end-preserved') + return + raise AssertionError('first migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, connection_context, kind, exit_behavior, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first-failure-and-original-end-preserved" in result.stdout From d71bfc96a0ffa8c426057b1d531cacaea88e1e3f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:31:05 +0000 Subject: [PATCH 061/181] Preserve native migration failures through owned cleanup Capture native failures inside owned work and public-close child tasks so their supervisors can clean up before rethrowing. Keep the first migration failure through context exit within its original cleanup deadline. Signed-off-by: zlareb1 --- server/core/pg.py | 14 ++- server/core/schema/pg_migrate.py | 75 +++++++++---- tests/unit/test_pg_close_owned.py | 75 +++++++++++++ tests/unit/test_pg_owned_fatal.py | 172 ++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_pg_owned_fatal.py diff --git a/server/core/pg.py b/server/core/pg.py index 5daeb33a55..1e51975dfb 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -392,16 +392,26 @@ async def close_owned(conn: Any, *, end: float) -> None: is a bounded fallback, not evidence that the backend has released its locks. The owner preserves an earlier operation failure when this cleanup fails. """ + + async def close() -> SystemExit | KeyboardInterrupt | None: + try: + await conn.close(timeout=max(0, end - time.monotonic())) + except (SystemExit, KeyboardInterrupt) as error: + # A native fatal in a child Task stops the loop before its owner can clean up. + return error + return None + task: asyncio.Task[Any] | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task = asyncio.create_task(conn.close(timeout=remaining)) + task = asyncio.create_task(close()) done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) if not done: raise TimeoutError("Postgres owned-session cleanup deadline expired") - task.result() + if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): + raise result except BaseException: _terminate(conn) if task is not None: diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 371f61a449..edc5cdb927 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -21,7 +21,7 @@ import logging import time from asyncio import sleep -from collections.abc import Callable +from collections.abc import Awaitable, Callable from contextlib import AbstractAsyncContextManager from pathlib import Path from typing import Any @@ -110,24 +110,61 @@ async def migrate( holds every read and write of the table behind it; the file is retried. For schema `loom` it marks the session and runs `cell_triggers.ensure` before the lock goes.""" files, indexes = _validated(mig_dir, schema) - async with (connect or pg.dedicated)() as conn: - task = asyncio.create_task(_migrate(conn, files, indexes, schema, lock_key)) - try: - return await asyncio.shield(task) - except BaseException: - end = time.monotonic() + pg._RELEASE_TIMEOUT_S - try: - await pg.close_owned(conn, end=end) - except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. - pass - if not task.done(): - task.cancel() - task.add_done_callback(pg._retrieve_task) - try: - await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - except BaseException: # noqa: BLE001 — preserve the original failure on repeated cancel. - pg._terminate(conn) - raise + cm = (connect or pg.dedicated)() + conn = await cm.__aenter__() + task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + try: + result = await asyncio.shield(task) + if isinstance(result, (SystemExit, KeyboardInterrupt)): + raise result + except BaseException as error: + end = time.monotonic() + pg._RELEASE_TIMEOUT_S + await _failed_migration(cm, conn, task, error, end) + raise + await cm.__aexit__(None, None, None) + return result + + +async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: + try: + return await work + except (SystemExit, KeyboardInterrupt) as error: + # Transport native failures through child Tasks without stopping their supervisor. + return error + + +async def _failed_migration( + cm: AbstractAsyncContextManager[Any], + conn: Any, + task: asyncio.Task[Any], + error: BaseException, + end: float, +) -> None: + try: + await pg.close_owned(conn, end=end) + except BaseException: # noqa: BLE001 — preserve the migration failure/cancellation. + pass + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + try: + await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) + except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. + pg._terminate(conn) + release = asyncio.create_task( + _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + ) + try: + done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) + if not done: + raise TimeoutError("migration context cleanup deadline expired") + release.result() + except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. + pg._terminate(conn) + finally: + if not release.done(): + release.cancel() + release.add_done_callback(pg._retrieve_task) def _validated( diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index fbbe778a5d..1ba7ab1f6a 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -142,3 +142,78 @@ async def fail(*_: Any) -> None: with pytest.raises(ValueError, match="migration failed"): await pg_migrate.migrate(tmp_path, connect=connect, schema="test") assert conn.aborted + + +async def test_successful_migration_still_reports_context_exit_failure( + monkeypatch, tmp_path +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + original = ValueError("context exit failed") + + @contextlib.asynccontextmanager + async def connect(): + yield Connection() + raise original + + async def applied(*_: Any) -> list[str]: + return ["001.sql"] + + monkeypatch.setattr(pg_migrate, "_migrate", applied) + with pytest.raises(ValueError) as error: + await pg_migrate.migrate(tmp_path, connect=connect, schema="test") + assert error.value is original + + +@pytest.mark.parametrize("cancel_work", [False, True]) +async def test_second_cancellation_during_migration_context_exit_keeps_first_failure( + monkeypatch, tmp_path, cancel_work +) -> None: + import contextlib + + from server.core.schema import pg_migrate + + conn = Connection() + conn.finished.set() + started, exiting = asyncio.Event(), asyncio.Event() + observed: list[BaseException] = [] + original = ValueError("migration failed") + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + except BaseException as error: + observed.append(error) + exiting.set() + await asyncio.Future() + raise + + async def work(*_: Any) -> list[str]: + started.set() + if cancel_work: + await asyncio.Future() + raise original + + monkeypatch.setattr(pg_migrate, "_migrate", work) + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.1) + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + await asyncio.wait_for(started.wait(), 1) + before = time.monotonic() + if cancel_work: + task.cancel("first cancellation") + await asyncio.wait_for(exiting.wait(), 1) + task.cancel("second cancellation") + with pytest.raises(BaseException) as result: + await asyncio.wait_for(task, 0.3) + assert time.monotonic() - before < 0.3 + assert result.value is observed[0] + assert ( + isinstance(result.value, asyncio.CancelledError) + if cancel_work + else result.value is original + ) + assert len(conn.timeouts) == 1 + assert conn.aborted diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py new file mode 100644 index 0000000000..683dbca782 --- /dev/null +++ b/tests/unit/test_pg_owned_fatal.py @@ -0,0 +1,172 @@ +"""Owned child tasks preserve native failures until their supervisor cleans up.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize( + "phase", + ["close", "migration", "migration_exit", "migration_ordinary_exit", "migration_exit_stall"], +) +def test_native_failure_reaches_the_owner_after_cleanup(kind: str, phase: str, tmp_path) -> None: + program = textwrap.dedent( + """ + import asyncio + import contextlib + import sys + import time + from pathlib import Path + from server.core import pg + from server.core.schema import pg_migrate + + kind, phase, path = sys.argv[1:] + fatal_type = SystemExit if kind == 'SystemExit' else KeyboardInterrupt + original = ValueError('original') if 'ordinary' in phase else fatal_type('original') + secondary = fatal_type('cleanup') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': + raise original + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + + @contextlib.asynccontextmanager + async def connect(): + try: + yield conn + finally: + events.append('exit') + if phase == 'migration_exit_stall': + await asyncio.Future() + if phase in {'migration_exit', 'migration_ordinary_exit'}: + raise secondary + + async def fail(*args): + events.append('work') + raise original + + async def main(): + pg_migrate._migrate = fail + pg._RELEASE_TIMEOUT_S = 0.05 + started = time.monotonic() + try: + if phase == 'close': + await pg.close_owned(conn, end=time.monotonic() + 1) + else: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert 'close' in events, events + if phase != 'close': + assert 'exit' in events, events + assert events.index('close') < events.index('exit'), events + else: + assert 'abort' in events, events + await asyncio.sleep(0) + print('owner-received-original-after-cleanup') + return + raise AssertionError('native failure was swallowed') + + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "owner-received-original-after-cleanup" in result.stdout + + +@pytest.mark.parametrize("connection_context", ["cell", "router"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt", "ValueError"]) +@pytest.mark.parametrize("exit_behavior", ["fatal", "stalled"]) +def test_actual_connection_context_keeps_first_failure_and_cleanup_end( + connection_context: str, kind: str, exit_behavior: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + import time + from pathlib import Path + import asyncpg + from server.core import pg + from server.core.schema import pg_migrate + from server.router import directory + + context, kind, exit_behavior, path = sys.argv[1:] + original = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt, + 'ValueError': ValueError}[kind]('original') + secondary = SystemExit('context failed') + events = [] + + class Connection: + async def close(self, **kwargs): + events.append('close') + if events.count('close') > 1: + if exit_behavior == 'stalled': + await asyncio.Future() + raise secondary + def terminate(self): + events.append('abort') + + conn = Connection() + async def raw(*args, **kwargs): + return conn + async def checked(*args): + pass + async def fail(*args): + raise original + + async def main(): + pg.settings.pg_dsn = "postgresql://unused/test" + pg._check_deployment = checked + pg.init_connection = checked + asyncpg.connect = raw + directory._require_dsn = lambda: 'unused-test-dsn' + pg._RELEASE_TIMEOUT_S = 0.05 + pg_migrate._migrate = fail + connect = None if context == 'cell' else directory._dedicated + started = time.monotonic() + try: + await pg_migrate.migrate(Path(path), connect=connect, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert time.monotonic() - started < 0.3, events + assert events.count('close') == 2, events + if exit_behavior == 'stalled': + assert 'abort' in events, events + await asyncio.sleep(0) + print('first-failure-and-original-end-preserved') + return + raise AssertionError('first migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, connection_context, kind, exit_behavior, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "first-failure-and-original-end-preserved" in result.stdout From 92d6d9be91565e9c68222ce89889462e6540786a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:33:56 +0000 Subject: [PATCH 062/181] Use common fatal transport for directory session close Signed-off-by: zlareb1 --- server/router/directory.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/server/router/directory.py b/server/router/directory.py index 42c48ff1fa..d4f3f62583 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -916,21 +916,6 @@ async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFa return error.fatal -@dataclass(frozen=True) -class _ControlConnection: - conn: Any - - async def close(self, **kwargs: Any) -> None: - try: - await self.conn.close(**kwargs) - except (SystemExit, KeyboardInterrupt) as error: - # pg.close_owned creates its own public-close child. - raise _ControlInterrupted(_ControlFatal(error)) from None - - def terminate(self) -> None: - pg._terminate(self.conn) - - class _ControlOwner: """Own only this operator attempt's proxy, through close and pool release.""" @@ -963,9 +948,7 @@ async def close(self) -> None: if self.conn is None: return if self.close_task is None: - self.close_task = self.start( - lambda: pg.close_owned(_ControlConnection(self.conn), end=self.end) - ) + self.close_task = self.start(lambda: pg.close_owned(self.conn, end=self.end)) try: await self.wait(self.close_task, self.end) except BaseException: From a13eb96b9d6a2be94268ed53471c620cc18896c5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:43:33 +0000 Subject: [PATCH 063/181] Construct migration cleanup work inside its error capture A custom context can raise before returning its exit awaitable. Defer that invocation to the owned child so it cannot replace the first migration error. Signed-off-by: zlareb1 --- server/core/schema/pg_migrate.py | 10 +++--- tests/unit/test_pg_owned_fatal.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index edc5cdb927..1722da5a9b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -112,7 +112,9 @@ async def migrate( files, indexes = _validated(mig_dir, schema) cm = (connect or pg.dedicated)() conn = await cm.__aenter__() - task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + task = asyncio.create_task( + _owned_result(lambda: _migrate(conn, files, indexes, schema, lock_key)) + ) try: result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): @@ -125,9 +127,9 @@ async def migrate( return result -async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: +async def _owned_result[T](work: Callable[[], Awaitable[T]]) -> T | SystemExit | KeyboardInterrupt: try: - return await work + return await work() except (SystemExit, KeyboardInterrupt) as error: # Transport native failures through child Tasks without stopping their supervisor. return error @@ -152,7 +154,7 @@ async def _failed_migration( except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. pg._terminate(conn) release = asyncio.create_task( - _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) try: done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py index 683dbca782..dcf77f560e 100644 --- a/tests/unit/test_pg_owned_fatal.py +++ b/tests/unit/test_pg_owned_fatal.py @@ -170,3 +170,58 @@ async def main(): ) assert result.returncode == 0, result.stdout + result.stderr assert "first-failure-and-original-end-preserved" in result.stdout + + +@pytest.mark.parametrize("exit_error", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_synchronous_context_exit_failure_keeps_original_migration_error( + exit_error: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + from pathlib import Path + from server.core.schema import pg_migrate + + kind, path = sys.argv[1:] + original = ValueError('original migration error') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('context factory failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + def terminate(self): + events.append('abort') + class Context: + async def __aenter__(self): + return Connection() + def __aexit__(self, kind, value, traceback): + assert value is original + events.append('exit') + raise secondary + async def fail(*args): + raise original + async def main(): + pg_migrate._migrate = fail + try: + await pg_migrate.migrate(Path(path), connect=Context, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[:2] == ['close', 'exit'], events + await asyncio.sleep(0) + print('synchronous-context-failure-contained') + return + raise AssertionError('migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, exit_error, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "synchronous-context-failure-contained" in result.stdout From 0410333de02afbf47bc4e182c5ca6dce7c88fcfe Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:45:20 +0000 Subject: [PATCH 064/181] Validate serving configuration and serialize fresh delivery rows Build required timestamps through the first-version helper and check retained launch evidence against the owned serving process and live fingerprints. Normalize accepted URLs and retain failed HTTP status and elapsed samples. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 242 +++++++++- tests/unit/test_bench_cell_trigger_gate.py | 491 ++++++++++++++++++++- 2 files changed, 712 insertions(+), 21 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 5bd3bfd3db..7321583095 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -15,6 +15,8 @@ import math import os import re +import socket +import stat import statistics import subprocess import sys @@ -241,6 +243,145 @@ def _source(fixture: Fixture) -> dict[str, Any]: return {"head": head, "tree": tree, "clean": True} +_PUBLIC_WORKLOAD = frozenset( + { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } +) + + +def _workload(config: Any) -> dict[str, Any]: + return { + key: value + for key, value in _configuration(config).items() + if (key.startswith("pg_") and key != "pg_dsn") + or key.startswith("memory_conflict") + or key in _PUBLIC_WORKLOAD + } + + +def _receipt(path: str) -> tuple[dict[str, Any], str]: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as source: + metadata = os.fstat(source.fileno()) + _require( + stat.S_ISREG(metadata.st_mode) and metadata.st_uid == os.geteuid(), + "owned regular receipt", + ) + raw = source.read(65_537) + _require(len(raw) <= 65_536, "receipt size") + value = json.loads(raw) + _require(isinstance(value, dict), "receipt object") + return value, hashlib.sha256(raw).hexdigest() + + +def _owned_server(server: dict[str, Any]) -> None: + from server.core.provider_process import _start_ticks + + _require( + set(server) + == {"pid", "start_ticks", "boot_id", "uid", "cwd", "health_instance", "instance_id"}, + "launch server fields", + ) + pid = server["pid"] + _require( + type(pid) is int + and pid > 1 + and type(server["uid"]) is int + and server["uid"] == os.geteuid(), + "launch process ownership", + ) + _require( + isinstance(server["start_ticks"], str) and server["start_ticks"].isdigit(), + "launch process birth", + ) + root = Path(__file__).resolve().parents[1] + _require( + server["cwd"] == str(root) and server["health_instance"] == f"{socket.gethostname()}:{pid}", + "launch process source/hostname/PID", + ) + proc = Path("/proc") / str(pid) + _require(proc.stat().st_uid == os.geteuid(), "actual process ownership") + birth = _start_ticks(pid) + _require(birth == server["start_ticks"], "actual process birth") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + _require( + boot == server["boot_id"] and os.readlink(proc / "cwd") == str(root), + "actual process boot/source", + ) + _require(_start_ticks(pid) == birth, "stable process birth") + + +def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str, Any]: + from server.core import deployment_config + + receipt, digest = _receipt(args.launch_manifest) + _require(digest == fixture.launch_manifest_sha256, "launch receipt hash") + _require( + set(receipt) == {"schema", "server", "source", "configuration_artifact"} + and type(receipt["schema"]) is int + and receipt["schema"] == 1, + "launch receipt schema", + ) + server, source, artifact = ( + receipt[key] for key in ("server", "source", "configuration_artifact") + ) + _require( + isinstance(server, dict) and isinstance(source, dict) and isinstance(artifact, dict), + "launch receipt objects", + ) + _require( + source == {"head": fixture.build_commit, "tree": fixture.source_tree}, + "launch source identity", + ) + _require(server.get("instance_id") == fixture.instance_id, "launch server UUID") + _require( + set(artifact) == {"path", "sha256"} + and isinstance(artifact["path"], str) + and Path(artifact["path"]).is_absolute(), + "configuration artifact path", + ) + configuration, artifact_digest = _receipt(artifact["path"]) + _require(artifact_digest == artifact["sha256"], "configuration artifact hash") + _require( + set(configuration) == {"schema", "configuration_sha256", "workload", "fingerprints"} + and type(configuration["schema"]) is int + and configuration["schema"] == 1, + "configuration artifact schema", + ) + _require( + configuration["configuration_sha256"] + == fixture.configuration_sha256 + == _digest(_configuration(config)), + "launch effective configuration", + ) + _require( + _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" + ) + fingerprints = deployment_config.fingerprints(config) + _require(configuration["fingerprints"] == fingerprints, "launch configuration fingerprints") + _owned_server(server) + return { + "health_instance": server["health_instance"], + "fingerprints": fingerprints, + "launch_sha256": digest, + "configuration_artifact_sha256": artifact_digest, + "birth_sha256": _digest(server), + } + + class Artifact: def __init__(self, path: str): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) @@ -302,10 +443,14 @@ async def close(self) -> None: await self.client.close() -async def _health(args: argparse.Namespace, fixture: Fixture) -> None: +async def _health( + args: argparse.Namespace, fixture: Fixture, launch: Mapping[str, Any] | None = None +) -> None: + from server.settings import settings + async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: - health = await client.get(f"{args.base_url}/v1/health") - deployment = await client.get(f"{args.base_url}/v1/health/deployment") + health = await client.get(f"{args.base_url.rstrip('/')}/v1/health") + deployment = await client.get(f"{args.base_url.rstrip('/')}/v1/health/deployment") _require(health.status_code == deployment.status_code == 200, "health HTTP status") h, d = health.json(), deployment.json() _require( @@ -328,6 +473,18 @@ async def _health(args: argparse.Namespace, fixture: Fixture) -> None: "serving provider/schema", ) _require(h.get("storage_authority") == "postgres", "serving authority") + for field in ("llm_provider", "llm_model", "embedding_model"): + _require(h.get(field) == getattr(settings, field), "serving " + field) + _require( + h.get("llm_provider") == "fake" + and type(h.get("embedding_dim_configured")) is int + and h["embedding_dim_configured"] == settings.embedding_dim + and type(h.get("embedding_dim_probed")) is int + and h["embedding_dim_probed"] == settings.embedding_dim, + "serving embedding dimensions/fake provider", + ) + launch = launch if launch is not None else _launch(args, fixture, settings) + _require(h.get("instance") == launch["health_instance"], "serving owned process") def _object(value: Any) -> dict[str, Any]: @@ -345,6 +502,7 @@ async def _proof( fixture.validate(args, settings) source = _source(fixture) + launch = _launch(args, fixture, settings) actual = await conn.fetchrow( "SELECT current_database() AS db, inet_server_port() AS port, " "pg_is_in_recovery() AS recovery" @@ -391,9 +549,15 @@ async def _proof( len(live) == 1 and live[0]["instance_id"] == fixture.instance_id and live[0]["build_commit"] == fixture.build_commit + and live[0].get("serving") == 1 + and live[0].get("host") == socket.gethostname() and not live[0]["workers_lease"], "exclusive serving source", ) + _require( + all(live[0].get(key) == value for key, value in launch["fingerprints"].items()), + "serving configuration fingerprints", + ) beats = await ch.query_all( "SELECT instance_id, cell, tenancy_mode, fingerprint FROM " "loom.cell_heartbeats WHERE instance_id = {id:String}", @@ -414,7 +578,7 @@ async def _proof( "serving fingerprint", ) await _debt(conn, 0) - await _health(args, fixture) + await _health(args, fixture, launch) if native: await cell.refresh() _require(not cell.cli_gate_reason(), "native fresh cell acceptance") @@ -425,6 +589,7 @@ async def _proof( "fingerprint_sha256": _digest(current), "configuration_sha256": fixture.configuration_sha256, "source": source, + "launch": launch, } @@ -610,9 +775,10 @@ async def _memory_set( key = f"{prefix}-{i}" value = _payload(key) started = time.perf_counter_ns() + response = None try: response = await client.post( - f"{args.base_url}/v1/memory.set", + f"{args.base_url.rstrip('/')}/v1/memory.set", json={ "org": args.org, "namespace": args.namespace, @@ -658,7 +824,29 @@ async def _memory_set( rows.append(row) emit("request", arm=arm, index=i, status=200, **asdict(row)) except BaseException as error: - emit("request_error", arm=arm, index=i, key=key, error_type=type(error).__name__) + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + code = "" + if response is not None: + try: + envelope = response.json() + candidate = envelope.get("error", {}).get("code", "") + if isinstance(candidate, str) and re.fullmatch( + r"[A-Z][A-Z0-9_]{0,63}", candidate + ): + code = candidate + except (ValueError, TypeError, AttributeError): + pass + emit( + "request_error", + arm=arm, + index=i, + key=key, + measured=i >= WARMUP, + elapsed_ms=elapsed, + status=response.status_code if response is not None else None, + code=code, + error_type=type(error).__name__, + ) raise return rows @@ -676,8 +864,10 @@ async def _build_block(args: argparse.Namespace, batch: int) -> Block: from server.memory.store import pg_memory, pg_publish from server.memory.store.rows import ( MemoryRow, + build_first_version_row, build_memory_version_row, next_version, + written_now, ) _planned_keys(batch) @@ -690,18 +880,21 @@ async def _build_block(args: argparse.Namespace, batch: int) -> Block: stored = json.dumps(value, sort_keys=True, separators=(",", ":")) row = build_memory_version_row( MemoryRow.from_snapshot( - { - "org": args.org, - "namespace": args.namespace, - "agent": "bench", - "memory_key": key, - "memory_type": "semantic", - "value": stored, - "version": next_version(), - "content_sha256": sha256_value(value), - "actor": SUBJECT, - "source_class": "asserted", - } + build_first_version_row( + { + "org": args.org, + "namespace": args.namespace, + "agent": "bench", + "memory_key": key, + "memory_type": "semantic", + "value": stored, + "version": next_version(), + "content_sha256": sha256_value(value), + "actor": SUBJECT, + "source_class": "asserted", + }, + now=written_now(), + ) ) ) row = pg_memory.decode_row(pg_memory.encode_row(row)) @@ -1166,7 +1359,16 @@ def main() -> int: from server.wiring import wire parser = argparse.ArgumentParser(description=__doc__) - for name in ("base-url", "pg-dsn", "token", "org", "namespace", "fixture-manifest", "output"): + for name in ( + "base-url", + "pg-dsn", + "token", + "org", + "namespace", + "fixture-manifest", + "launch-manifest", + "output", + ): parser.add_argument("--" + name, required=True) args = parser.parse_args() artifact = None @@ -1176,11 +1378,13 @@ def main() -> int: fixture = Fixture.read(args.fixture_manifest) total = fixture.validate(args, settings) source = _source(fixture) + launch = _launch(args, fixture, settings) artifact = Artifact(args.output) artifact.emit( "configuration", manifest=asdict(fixture), source=source, + launch=launch, manifest_sha256=_digest(asdict(fixture)), calls=CALLS, warmup=WARMUP, diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a27f9560f3..31986733a8 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -6,6 +6,7 @@ import asyncio import dataclasses import json +import socket import uuid from contextlib import asynccontextmanager from types import SimpleNamespace @@ -35,6 +36,7 @@ def args(**overrides): "token": "secret-token", "org": "org", "namespace": "ns", + "launch_manifest": "unused", **overrides, } ) @@ -51,11 +53,27 @@ def fixture_config(): background_workers="off", storage_authority="postgres", embedding_provider="fake", + embedding_model="fake-ngram-16", + embedding_dim=16, + embedding_base_url="", + embedding_send_dimensions=False, llm_provider="fake", + llm_model="fake", + jwt_secret="test-only-root", + deployment=False, + instance_heartbeat_seconds=10, build_commit="a" * 40, pg_delivery_batch_rows=500, ) - config.model_dump = lambda **_: {k: v for k, v in vars(config).items() if k != "model_dump"} + + def dump(**kwargs): + return { + k: v + for k, v in vars(config).items() + if k != "model_dump" and ("include" not in kwargs or k in kwargs["include"]) + } + + config.model_dump = dump manifest = gate.Fixture( base_url=args().base_url, pg_host="127.0.0.1", @@ -609,7 +627,16 @@ async def live(*_args, **_kw): return ( [] if fault == "competing_worker" - else [{"instance_id": "instance", "build_commit": "a" * 40, "workers_lease": ""}] + else [ + { + "instance_id": "instance", + "build_commit": "a" * 40, + "workers_lease": "", + "serving": 1, + "host": socket.gethostname(), + **fingerprints, + } + ] ) async def health(*_args): @@ -623,6 +650,10 @@ async def health(*_args): monkeypatch.setattr(gate, "_close_pg", close) monkeypatch.setattr(gate, "_health", health) monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + from server.core import deployment_config + + fingerprints = deployment_config.fingerprints(config) + monkeypatch.setattr(gate, "_launch", lambda *_: {"fingerprints": fingerprints}) monkeypatch.setattr(instances, "read_live", live) monkeypatch.setattr(cell, "mine", lambda: {}) monkeypatch.setattr(cell, "matches", lambda mine, value: value is not None) @@ -744,6 +775,7 @@ def test_final_verdict_follows_native_cleanup_and_artifact_close( monkeypatch.setattr(server.wiring, "wire", lambda: order.append("wire")) monkeypatch.setattr(gate, "_preflight", AsyncMock()) monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) def native(*_a, **_kw): order.append("native_cleanup") @@ -969,3 +1001,458 @@ async def close(owned): with pytest.raises(TimeoutError): await gate._cleanup(Stuck(), conn, artifact) assert conn.closed + + +@pytest.mark.asyncio +async def test_review_delivery_timestamps_pass_real_clickhouse_serialization(monkeypatch): + import logging + + from clickhouse_connect.datatypes.registry import get_from_name + from clickhouse_connect.driver.insert import InsertContext + + import server.embedding + from server.core.ch import _insert_data + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + for col in ("created_at", "updated_at", "last_reinforced_at"): + actual_data = _insert_data(block.rows, [col], ["DateTime"]) + kind = get_from_name("DateTime") + context = InsertContext("loom.memory", [col], [kind]) + context.start_column(col) + serialized = bytearray() + kind.write_column([row[0] for row in actual_data], serialized, context) + assert len(serialized) == 4 * len(block.rows) + assert all( + row["created_at"] == row["updated_at"] == row["last_reinforced_at"] for row in block.rows + ) + + +@pytest.mark.asyncio +async def test_review_trailing_slash_is_normalized_for_actual_health_and_api_routes(monkeypatch): + from starlette.responses import JSONResponse + from starlette.routing import Route, Router + + fixture, _ = fixture_config() + paths = [] + + async def health(request): + paths.append(request.url.path) + return JSONResponse({}) + + app = Router( + routes=[ + Route("/v1/health", health), + Route("/v1/health/deployment", health), + Route("/v1/memory.set", health, methods=["POST"]), + ] + ) + transport = httpx.ASGITransport(app=app) + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + with pytest.raises(gate.Invalid): + await gate._health(args(base_url=fixture.base_url + "/"), fixture) + with pytest.raises(gate.Invalid): + await gate._memory_set(args(base_url=fixture.base_url + "/")) + assert paths == ["/v1/health", "/v1/health/deployment", "/v1/memory.set"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status,code", [(302, ""), (503, "PUBLICATION_PENDING")]) +async def test_review_failed_request_keeps_raw_status_elapsed_and_classification( + monkeypatch, status, code +): + async def send(self, request, **kwargs): + return httpx.Response( + status, json={"error": {"code": code, "message": "secret-token"}}, request=request + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([10_000_000, 17_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + events = [] + with pytest.raises(gate.Invalid): + await gate._memory_set(args(), emit=lambda kind, **record: events.append((kind, record))) + assert len(events) == 1 and events[0][0] == "request_error" + record = events[0][1] + assert record["status"] == status and record["code"] == code + assert record["elapsed_ms"] == 7 and record["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "", + "settings_fp", + "keys_fp", + "config_fp", + "llm_provider", + "llm_model", + "embedding_model", + "embedding_dim_configured", + "embedding_dim_probed", + "host", + "instance", + "uuid", + ], +) +async def test_review_composed_proof_matches_actual_server_fingerprints_and_health( + monkeypatch, tmp_path, fault +): + import server.settings + from server.core import cell, deployment_config + + fixture, config = fixture_config() + fixture, launch_path, _, launch, _ = launch_files(tmp_path, fixture, config) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + mine = cell.mine() + mine["deployment_id"] = fixture.deployment_id + fingerprints = deployment_config.fingerprints(config) + + class PG(ProofPG): + async def fetchrow(self, sql, *params): + if "FROM loom.cell_self" in sql: + return {"current_fp": json.dumps(cell.as_value(mine)), "next_fp": None} + return await super().fetchrow(sql, *params) + + class CH(ProofCH): + async def query_all(self, sql, params=None): + if "FROM loom.instances" in sql: + return [ + { + "instance_id": fixture.instance_id, + "build_commit": fixture.build_commit, + "workers_lease": "", + "serving": 1, + "host": "other" if fault == "host" else socket.gethostname(), + **fingerprints, + **({fault: "wrong"} if fault.endswith("_fp") else {}), + } + ] + if "FROM loom.cell_heartbeats" in sql: + return [ + { + "cell": fixture.cell, + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + {**cell.as_value(mine), "serves": True, "marked": True} + ), + } + ] + return await super().query_all(sql, params) + + async def send(self, request, **kwargs): + body = ( + {"deployment_id": fixture.deployment_id} + if request.url.path.endswith("/deployment") + else { + "build_commit": fixture.build_commit, + "instance": "other:4242" + if fault == "instance" + else launch["server"]["health_instance"], + "deployment_config": { + "instance_id": "other" if fault == "uuid" else fixture.instance_id + }, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + **( + {fault: 999 if "dim" in fault else "wrong"} + if fault and not fault.endswith("_fp") + else {} + ), + } + ) + return httpx.Response(200, json=body, request=request) + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(cell, "mine", lambda: dict(mine)) + monkeypatch.setattr(httpx.AsyncClient, "send", send) + if fault: + with pytest.raises(gate.Invalid): + await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + else: + proof = await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) + assert proof["configuration_sha256"] == fixture.configuration_sha256 + + +def launch_files(tmp_path, fixture, config): + import os + import socket + + from server.core import deployment_config + + public = { + k: v + for k, v in config.model_dump().items() + if k.startswith("pg_") + and k != "pg_dsn" + or k.startswith("memory_conflict") + or k + in { + "storage_authority", + "background_workers", + "embedding_provider", + "embedding_model", + "embedding_dim", + "embedding_send_dimensions", + "embedding_worker_enabled", + "embedding_cache_shared", + "embedding_cache_shared_max_chars", + "embedding_cache_size", + "embedding_cache_ttl_s", + "llm_provider", + "llm_model", + } + } + configuration = { + "schema": 1, + "configuration_sha256": fixture.configuration_sha256, + "workload": public, + "fingerprints": deployment_config.fingerprints(config), + } + artifact_path = tmp_path / "configuration.json" + artifact_bytes = json.dumps(configuration).encode() + artifact_path.write_bytes(artifact_bytes) + source_path = str(__import__("pathlib").Path(gate.__file__).resolve().parents[1]) + launch = { + "schema": 1, + "server": { + "pid": 4242, + "start_ticks": "123", + "boot_id": "fake-boot-id", + "uid": os.geteuid(), + "cwd": source_path, + "health_instance": f"{socket.gethostname()}:4242", + "instance_id": fixture.instance_id, + }, + "source": {"head": fixture.build_commit, "tree": fixture.source_tree}, + "configuration_artifact": { + "path": str(artifact_path), + "sha256": gate.hashlib.sha256(artifact_bytes).hexdigest(), + }, + } + launch_path = tmp_path / "launch.json" + launch_bytes = json.dumps(launch).encode() + launch_path.write_bytes(launch_bytes) + fixture = dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_bytes).hexdigest() + ) + return fixture, launch_path, artifact_path, launch, configuration + + +@pytest.mark.parametrize( + "fault", + [ + "", + "launch_hash", + "artifact_hash", + "config", + "workload", + "workload_type", + "fingerprints", + "source", + "instance", + "symlink", + ], +) +def test_launch_receipt_binds_named_owned_process_settings_and_nonsecret_artifact( + monkeypatch, tmp_path, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda server: checked.append(server), raising=False) + if fault in {"config", "workload", "workload_type", "fingerprints"}: + if fault == "config": + configuration["configuration_sha256"] = "0" * 64 + if fault == "workload": + configuration["workload"]["pg_delivery_batch_rows"] = 501 + if fault == "workload_type": + configuration["workload"]["pg_delivery_batch_rows"] = 500.0 + if fault == "fingerprints": + configuration["fingerprints"]["keys_fp"] = "wrong" + blob = json.dumps(configuration).encode() + artifact_path.write_bytes(blob) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(blob).hexdigest() + if fault == "artifact_hash": + launch["configuration_artifact"]["sha256"] = "0" * 64 + if fault == "source": + launch["source"]["tree"] = "0" * 40 + if fault == "instance": + launch["server"]["instance_id"] = "other" + blob = json.dumps(launch).encode() + path.write_bytes(blob) + fixture = dataclasses.replace( + fixture, + launch_manifest_sha256="0" * 64 + if fault == "launch_hash" + else gate.hashlib.sha256(blob).hexdigest(), + ) + if fault == "symlink": + link = tmp_path / "link.json" + link.symlink_to(path) + path = link + if fault: + with pytest.raises((gate.Invalid, OSError)): + gate._launch(args(launch_manifest=str(path)), fixture, config) + else: + result = gate._launch(args(launch_manifest=str(path)), fixture, config) + assert result["health_instance"] == launch["server"]["health_instance"] + assert result["fingerprints"] == configuration["fingerprints"] + assert checked == [launch["server"]] + + +@pytest.mark.parametrize( + "fault", + ["", "uid", "actual_uid", "pid", "start", "reuse", "boot", "cwd", "stamp_pid", "stamp_host"], +) +def test_owned_server_reads_only_exact_named_birth_metadata(monkeypatch, tmp_path, fault): + import os + from pathlib import Path + + from server.core import provider_process + + fixture, config = fixture_config() + _, _, _, launch, _ = launch_files(tmp_path, fixture, config) + server = dict(launch["server"]) + if fault == "uid": + server["uid"] += 1 + if fault == "pid": + server["pid"] = True + if fault == "stamp_pid": + server["health_instance"] = f"{socket.gethostname()}:4243" + if fault == "stamp_host": + server["health_instance"] = "other:4242" + paths, births = [], [] + real_stat = Path.stat + + def metadata(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242" + paths.append(str(path)) + return SimpleNamespace(st_uid=os.geteuid() + int(fault == "actual_uid")) + return real_stat(path, **kwargs) + + def start(pid): + assert type(pid) is int and pid == 4242 + births.append(pid) + return "wrong" if fault == "start" or fault == "reuse" and len(births) == 2 else "123" + + real_read = Path.read_text + + def read(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/sys/kernel/random/boot_id" + paths.append(str(path)) + return "wrong" if fault == "boot" else "fake-boot-id" + return real_read(path, **kwargs) + + real_link = os.readlink + + def link(path, **kwargs): + if str(path).startswith("/proc"): + assert str(path) == "/proc/4242/cwd" + paths.append(str(path)) + return "other" if fault == "cwd" else server["cwd"] + return real_link(path, **kwargs) + + monkeypatch.setattr(Path, "stat", metadata) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(provider_process, "_start_ticks", start) + monkeypatch.setattr(os, "readlink", link) + if fault: + with pytest.raises(gate.Invalid): + gate._owned_server(server) + else: + gate._owned_server(server) + assert births == [4242, 4242] + assert paths == ["/proc/4242", "/proc/sys/kernel/random/boot_id", "/proc/4242/cwd"] + + +@pytest.mark.asyncio +async def test_failed_network_request_retains_elapsed_with_no_invented_status(monkeypatch): + async def send(self, request, **kwargs): + raise httpx.ConnectError("secret-token", request=request) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + clock = iter([1_000_000, 3_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(clock)) + records = [] + with pytest.raises(httpx.ConnectError): + await gate._memory_set(args(), emit=lambda kind, **record: records.append(record)) + assert len(records) == 1 and records[0]["elapsed_ms"] == 2 + assert records[0]["status"] is None and records[0]["code"] == "" + assert "secret-token" not in json.dumps(records) + + +def test_nonregular_receipt_open_is_nonblocking_before_type_check(monkeypatch): + import os + + def open_receipt(path, flags): + assert flags & os.O_NONBLOCK + assert flags & os.O_NOFOLLOW + raise gate.Invalid("nonregular receipt") + + monkeypatch.setattr(gate.os, "open", open_receipt) + with pytest.raises(gate.Invalid, match="nonregular"): + gate._receipt("named-owned-receipt") + + +def test_launch_refusal_precedes_any_native_or_store_setup(monkeypatch, tmp_path, capsys): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + fixture = dataclasses.replace(fixture, launch_manifest_sha256="0" * 64) + manifest = tmp_path / "fixture.json" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + arguments = vars(args(launch_manifest=str(launch_path))) + arguments.update(fixture_manifest=str(manifest), output=str(tmp_path / "output.jsonl")) + argv = ["driver"] + for name, value in arguments.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + + def forbidden(*_args, **_kwargs): + raise AssertionError("launch refusal must precede native/process setup") + + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", forbidden) + monkeypatch.setattr(gate, "_preflight", forbidden) + monkeypatch.setattr(server.wiring, "wire", forbidden) + monkeypatch.setattr(_ch, "run_connected", forbidden) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["reason"] == "launch receipt hash" + assert not (tmp_path / "output.jsonl").exists() From 8c0320060dd4709ac75772fcf1f6e99776e3efc9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:43:33 +0000 Subject: [PATCH 065/181] Construct migration cleanup work inside its error capture A custom context can raise before returning its exit awaitable. Defer that invocation to the owned child so it cannot replace the first migration error. Signed-off-by: zlareb1 (cherry picked from commit 613209b4ab0398601e9375d433d490a08b9dc110) --- server/core/schema/pg_migrate.py | 10 +++--- tests/unit/test_pg_owned_fatal.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index edc5cdb927..1722da5a9b 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -112,7 +112,9 @@ async def migrate( files, indexes = _validated(mig_dir, schema) cm = (connect or pg.dedicated)() conn = await cm.__aenter__() - task = asyncio.create_task(_owned_result(_migrate(conn, files, indexes, schema, lock_key))) + task = asyncio.create_task( + _owned_result(lambda: _migrate(conn, files, indexes, schema, lock_key)) + ) try: result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): @@ -125,9 +127,9 @@ async def migrate( return result -async def _owned_result[T](work: Awaitable[T]) -> T | SystemExit | KeyboardInterrupt: +async def _owned_result[T](work: Callable[[], Awaitable[T]]) -> T | SystemExit | KeyboardInterrupt: try: - return await work + return await work() except (SystemExit, KeyboardInterrupt) as error: # Transport native failures through child Tasks without stopping their supervisor. return error @@ -152,7 +154,7 @@ async def _failed_migration( except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. pg._terminate(conn) release = asyncio.create_task( - _owned_result(cm.__aexit__(type(error), error, error.__traceback__)) + _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) try: done, _ = await asyncio.wait({release}, timeout=max(0, end - time.monotonic())) diff --git a/tests/unit/test_pg_owned_fatal.py b/tests/unit/test_pg_owned_fatal.py index 683dbca782..dcf77f560e 100644 --- a/tests/unit/test_pg_owned_fatal.py +++ b/tests/unit/test_pg_owned_fatal.py @@ -170,3 +170,58 @@ async def main(): ) assert result.returncode == 0, result.stdout + result.stderr assert "first-failure-and-original-end-preserved" in result.stdout + + +@pytest.mark.parametrize("exit_error", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_synchronous_context_exit_failure_keeps_original_migration_error( + exit_error: str, tmp_path +) -> None: + program = textwrap.dedent( + """ + import asyncio + import sys + from pathlib import Path + from server.core.schema import pg_migrate + + kind, path = sys.argv[1:] + original = ValueError('original migration error') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('context factory failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + def terminate(self): + events.append('abort') + class Context: + async def __aenter__(self): + return Connection() + def __aexit__(self, kind, value, traceback): + assert value is original + events.append('exit') + raise secondary + async def fail(*args): + raise original + async def main(): + pg_migrate._migrate = fail + try: + await pg_migrate.migrate(Path(path), connect=Context, schema='test') + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[:2] == ['close', 'exit'], events + await asyncio.sleep(0) + print('synchronous-context-failure-contained') + return + raise AssertionError('migration failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, exit_error, str(tmp_path)], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "synchronous-context-failure-contained" in result.stdout From d27ff2e5e39bcdbd2c61e3d541a01a50d2e13d14 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:04:24 +0000 Subject: [PATCH 066/181] Close placement sessions before cancelling their work Signed-off-by: zlareb1 --- docs/architecture/cells.md | 9 +- server/cli/cell_control.py | 80 +++-- server/cli/cell_place.py | 52 +-- server/router/app.py | 2 +- .../integration/test_cell_control_cleanup.py | 12 +- tests/integration/test_cell_place_cleanup.py | 106 ++++++ tests/unit/test_cell_attach_cleanup.py | 10 +- tests/unit/test_cell_control_census.py | 8 +- tests/unit/test_cell_place.py | 164 +++++++++- tests/unit/test_cell_place_owner.py | 307 ++++++++++++++++++ 10 files changed, 672 insertions(+), 78 deletions(-) create mode 100644 tests/integration/test_cell_place_cleanup.py create mode 100644 tests/unit/test_cell_place_owner.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 4cdb16ecaf..379e91853c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -764,8 +764,13 @@ read-only scope, lists any tenancy row the directory does not place. session, requires a full enforced local cell state and the full accepted configuration fingerprint, and checks both again before inserting. A transactional durability message makes even a no-op insert flush an earlier primary-only tenancy row to the standby. - Opening the session, local statements, COMMIT and cleanup have client deadlines derived - from `LOOM_PG_STATEMENT_TIMEOUT_MS`. An unknown outcome names rerunning the same command; + Opening the session, initial checks and the local transaction through COMMIT each have + a separate client deadline derived from `LOOM_PG_STATEMENT_TIMEOUT_MS`; the directory + request keeps its own transport timeout. Each local phase reserves cleanup inside its + original deadline. On failure the shared control owner closes the exact backend before + cancelling or reaping its work, deferring repeated signals within that same deadline. + Successful exit and failures outside a local phase have a separate cleanup allowance + derived from the same setting. An unknown outcome names rerunning the same command; the permanent directory winner and idempotent local insert let that rerun finish. The route checks directory state; a direct operator API caller is trusted to check local Postgres mode as the CLI does. For an org the directory does not place yet it diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index c4360839ed..74d804fdb6 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -119,7 +119,8 @@ async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: class Owner: """One raw backend and one cleanup owner for a native control command.""" - def __init__(self) -> None: + def __init__(self, *, exit_s: float = pg._RELEASE_TIMEOUT_S) -> None: + self.exit_s = exit_s self.conn: Any = None self.phase_end: float | None = None self.close_end: float | None = None @@ -155,7 +156,7 @@ async def _open(self, what: str) -> None: async def close(self) -> None: if not self.closed: self.closed = True - self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + self.close_end = self.phase_end or (time.monotonic() + self.exit_s) if self.conn is None: return if self._close_task is None: @@ -163,12 +164,17 @@ async def close(self) -> None: _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) ) self._close_task.add_done_callback(pg._retrieve_task) + interrupted: asyncio.CancelledError | None = None try: - done, _ = await asyncio.wait( - {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) - ) - if not done: - raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + while not self._close_task.done(): + remaining = cast(float, self.close_end) - time.monotonic() + if remaining <= 0: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + try: + await asyncio.wait({self._close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Signals cannot abort the public cancellation already in flight. + interrupted = interrupted or error result = self._close_task.result() if isinstance(result, _Fatal): raise _ForwardFatal(result) @@ -176,7 +182,11 @@ async def close(self) -> None: pg._terminate(self.conn) if not self._close_task.done(): self._close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: for task in tasks: @@ -219,31 +229,43 @@ async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) + async def run[T]( + self, + opening: Callable[[], Awaitable[None]], + work: Callable[[], Awaitable[T]], + *, + opening_s: float, + ) -> T: + """Own one command; fatal guard failures leave only after bounded cleanup.""" + if self.conn is not None or self.phase_end is not None or self._tasks or self.closed: + raise _refused("a command needs a fresh control owner") + try: + await self.call(opening, total_s=opening_s) + result = await asyncio.shield(self._start(work)) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await self.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await self._stop(set(self._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise + try: + await self.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None + finally: + await self._stop(set(self._tasks)) + return result + async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: - """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + """Run native control with its existing opening, transaction and exit allowances.""" owner = Owner() - try: - await owner.call(lambda: owner._open(what), total_s=OPEN_S) - result = await asyncio.shield(owner._start(lambda: work(owner))) - if isinstance(result, _Fatal): - raise _ForwardFatal(result) - except BaseException as original: - try: - await owner.close() - except BaseException: # noqa: BLE001 — preserve the command's original failure. - pass - await owner._stop(set(owner._tasks)) - if isinstance(original, _ForwardFatal): - raise original.fatal.error from None - raise - try: - await owner.close() - except _ForwardFatal as interrupted: - raise interrupted.fatal.error from None - finally: - await owner._stop(set(owner._tasks)) - return result + return await owner.run(lambda: owner._open(what), lambda: work(owner), opening_s=OPEN_S) async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index 3820ecf2cc..573e0ff2fc 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -14,6 +14,7 @@ from server.cli import cell_attach_pg as apg from server.cli._ch import run_connected, writes from server.cli.cell_ch import Refused +from server.cli.cell_control import Owner from server.core import cell, pg from server.router.directory import PLACE_MAX_S from server.settings import settings @@ -88,18 +89,34 @@ async def _directory( async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: """Hold the operator lock through directory placement and an acknowledged local commit. Every local phase, including opening and closing the direct session, has a deadline.""" - cm = pg.dedicated(statement_timeout_ms=settings.pg_statement_timeout_ms) - conn = None - try: - async with asyncio.timeout(_bound_s()): - conn = await cm.__aenter__() - async with asyncio.timeout(_bound_s()): - if refusals := await apg.direct_refusals(conn, "place"): - raise Refused(refusals) - await apg.take_lock(conn, "place") - deployment, fingerprint = await _identity(conn, args.cell) + owner = Owner(exit_s=_bound_s()) + + async def opening() -> None: + owner.conn = await pg._connect_dedicated( + statement_timeout_ms=settings.pg_statement_timeout_ms + ) + if owner.closed: + await owner.close() + raise PlacementRefused("connection arrived after the placement attempt ended") + try: + await pg._check_deployment(owner.conn) + await pg.init_connection(owner.conn) + except Exception as error: + pg._raise_if_unavailable(error) + raise + + async def initial() -> tuple[str, dict[str, Any]]: + if refusals := await apg.direct_refusals(owner.conn, "place"): + raise Refused(refusals) + await apg.take_lock(owner.conn, "place") + return await _identity(owner.conn, args.cell) + + async def work() -> dict[str, str]: + deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) await _directory(router, args.cell, args.org, deployment, fingerprint) - async with asyncio.timeout(_bound_s()): + + async def local() -> None: + conn = owner.conn tx = conn.transaction() await tx.start() if not await apg.lock_held(conn): @@ -114,14 +131,11 @@ async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: # A no-op insert must also flush a primary-only predecessor to the standby. await conn.execute(pg._DURABLE_MESSAGE) await tx.commit() - finally: - if conn is not None: - try: - async with asyncio.timeout(_bound_s()): - await cm.__aexit__(None, None, None) - finally: - conn.terminate() - return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + await owner.call(local, total_s=_bound_s()) + return {"org": args.org, "cell": args.cell, "deployment_id": deployment} + + return await owner.run(opening, work, opening_s=_bound_s()) def cmd_place(args: argparse.Namespace) -> int: diff --git a/server/router/app.py b/server/router/app.py index 6cf6d9e729..212e87f737 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -320,7 +320,7 @@ def refusal(locked: directory.Cell) -> str: async def operator_place(self, request: Request) -> JSONResponse: """Place with the operator credential. The CLI checks its local Postgres mode; this route trusts that check and verifies the directory's locked identity.""" - body = await self._operator_body(request) + body = await self._operator_body(request, MAX_BODY_BYTES) if isinstance(body, JSONResponse): return body org = cell_state.named_org(body.get("org")) diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index e057fef323..8f68939b62 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -105,6 +105,8 @@ async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( real_close = asyncpg.Connection.close observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} + close_interrupted = asyncio.Event() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) async def read(conn, **_): backend["conn"], backend["pid"] = conn, conn.get_server_pid() @@ -115,7 +117,11 @@ async def delayed_close(self, **kwargs): await real_close(self, **kwargs) if self is backend.get("conn"): close_started.set() - await close_return.wait() + try: + await close_return.wait() + except asyncio.CancelledError: + close_interrupted.set() + raise monkeypatch.setattr(cell_prepare, "_read_rows", read) monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) @@ -128,6 +134,10 @@ async def delayed_close(self, **kwargs): task.cancel("first native cancellation") await asyncio.wait_for(close_started.wait(), 1) task.cancel("second cancellation") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not close_interrupted.is_set(), "second signal cannot interrupt public close" + assert not task.done() with pytest.raises(asyncio.CancelledError, match="first native cancellation"): await asyncio.wait_for(task, 0.5) await released(native_database, backend["pid"]) diff --git a/tests/integration/test_cell_place_cleanup.py b/tests/integration/test_cell_place_cleanup.py new file mode 100644 index 0000000000..3bbb5acc07 --- /dev/null +++ b/tests/integration/test_cell_place_cleanup.py @@ -0,0 +1,106 @@ +"""Placement cleanup observes the actual marked backend through interrupted work.""" + +from __future__ import annotations + +import asyncio +import contextlib +import time + +import asyncpg +import pytest + +from server.cli import cell_place +from server.core import pg +from tests.integration import test_cell_place as placement + + +@pytest.fixture +async def stores(monkeypatch): + async for value in placement.stores.__wrapped__(monkeypatch): + yield value + + +async def waiting(observer, pid): + async with asyncio.timeout(3): + while True: + if await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event='PgSleep')", + pid, + ): + return + await asyncio.sleep(0.01) + + +async def assert_released(observer, pid): + assert not await observer.fetchval("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", pid) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ) + + +@pytest.mark.parametrize("interrupt", ["deadline", "cancel", "second_cancel"]) +async def test_placement_owner_closes_before_cancelling_actual_rollback( + stores, monkeypatch, interrupt +): + # Only the exact ROLLBACK wire operation is delayed. The transaction context, + # sleeping query, session lock, observer and close are the installed real driver. + original_identity, execute = cell_place._identity, asyncpg.Connection.execute + known = asyncio.Event() + captured = {} + queries = [] + + async def stalled_rollback(conn, sql, *args, **kwargs): + if conn is captured.get("conn") and sql == "ROLLBACK;": + query = asyncio.create_task(execute(conn, "SELECT pg_sleep(10)")) + queries.append(query) + while not query.done(): + try: + await asyncio.shield(query) + except asyncio.CancelledError: + continue + query.result() + return await execute(conn, sql, *args, **kwargs) + + async def identity(conn, name): + await original_identity(conn, name) + captured["conn"] = conn + known.set() + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(193789)") + raise ValueError("placement body failed before rollback") + + monkeypatch.setattr(cell_place, "_identity", identity) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + monkeypatch.setattr(asyncpg.Connection, "execute", stalled_rollback) + observer = await asyncpg.connect(stores["first"]) + task = asyncio.create_task(placement._local(stores, monkeypatch, "first")) + try: + await asyncio.wait_for(known.wait(), 3) + conn = captured["conn"] + pid = conn.get_server_pid() + await waiting(observer, pid) + started = time.monotonic() + if interrupt != "deadline": + task.cancel() + if interrupt == "second_cancel": + # A second signal arrives while the same operation is still active. + await asyncio.sleep(0) + task.cancel() + done, _ = await asyncio.wait({task}, timeout=0.8) + assert done, "placement must close its backend before waiting for rollback unwind" + with pytest.raises(TimeoutError if interrupt == "deadline" else asyncio.CancelledError): + await task + assert time.monotonic() - started < 0.8 + await assert_released(observer, pid) + finally: + # Exact owned test resources only; even a deliberately failing old owner + # cannot leave its query or scratch database alive after the regression. + if "conn" in captured and not captured["conn"].is_closed(): + with contextlib.suppress(Exception): + await captured["conn"].close(timeout=1) + if not task.done(): + task.cancel() + await asyncio.gather(task, *queries, return_exceptions=True) + await observer.close(timeout=1) + pg.expect_deployment(None) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 11bf1bf094..a018784d21 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -8,7 +8,6 @@ import pytest from server.cli import cell_attach, cell_control -from server.core import pg from tests.unit.cell_control_owner import owner from tests.unit.test_cell_control import snapshot @@ -52,6 +51,9 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + def is_closed(self): + return False + async def work(): entered.set() try: @@ -78,7 +80,7 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() conn.terminate = lambda: None @@ -88,8 +90,8 @@ async def close(**_): await asyncio.Future() conn.close.side_effect = close - owned = owner(conn) - monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + owned = cell_control.Owner(exit_s=0.03) + owned.conn = conn started = time.monotonic() with pytest.raises(TimeoutError): await owned.close() diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 45b8e85baa..7fc1fb2e2b 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -143,7 +143,7 @@ def test_cell_control_migrations_are_classified(): ("register", "INSERT INTO", "loom_router.audit"): 1, ("_place_one", "INSERT INTO", "loom_router.placements"): 1, ("_place_one", "UPDATE", "loom_router.cells"): 1, - ("_place_one", "INSERT INTO", "loom_router.audit"): 1, + ("place", "INSERT INTO", "loom_router.audit"): 1, ("_audit", "INSERT INTO", "loom_router.audit"): 1, }, } @@ -493,9 +493,9 @@ def test_reads_registration_and_immutable_placement_are_explicit_exceptions(): # Operator reads may audit, but cannot bypass control guards for a state write. assert any(call.startswith("_audit(") for call in calls(found.functions["attach_read"])) assert "_rerun_refusal(held, reg)" in calls(found.functions["register"]) - assert "_place_one(conn, org, cell_name, _cell(locked), refusal)" in calls( - found.functions["place"] - ) + assert "_place_one(conn, org, cell_name, candidate, refusal)" in calls(found.functions["place"]) + assert "_cell(locked)" in calls(found.functions["place"]) + assert "verify(candidate)" in calls(found.functions["place"]) assert "refusal(candidate)" in calls(found.functions["_place_one"]) for name in ("set_limit", "rotate_credential"): assert not any(call.startswith("claim(") for call in calls(found.functions[name])) diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py index 5f8deeaf0e..26a06748a1 100644 --- a/tests/unit/test_cell_place.py +++ b/tests/unit/test_cell_place.py @@ -125,6 +125,7 @@ def __init__(self): self.calls = [] self.committed = False self.terminated = False + self.closed = False self.stall_commit = False self.stall_close = False self.fail_insert = False @@ -151,9 +152,18 @@ async def execute(self, sql, *args): raise OSError("local insert failed") return "INSERT 0 0" + async def close(self, **kwargs): + self.calls.append("close") + if self.stall_close: + await asyncio.Event().wait() + self.closed = True + def terminate(self): self.terminated = True + def is_closed(self): + return self.closed or self.terminated + @pytest.fixture def local(monkeypatch): @@ -175,13 +185,12 @@ async def fetchrow(sql): conn.fetchval = fetchval conn.fetchrow = fetchrow - @asynccontextmanager - async def dedicated(**kw): - try: - yield conn - finally: - if conn.stall_close: - await asyncio.Event().wait() + async def connect(**kw): + assert kw == {"statement_timeout_ms": settings.pg_statement_timeout_ms} + return conn + + async def initialize(c): + assert c is conn async def direct(c, what): return [] @@ -195,7 +204,9 @@ async def held(c): async def directory_call(*a): conn.calls.append("directory") - monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(pg, "_connect_dedicated", connect) + monkeypatch.setattr(pg, "_check_deployment", initialize) + monkeypatch.setattr(pg, "init_connection", initialize) monkeypatch.setattr(cell, "read_state_if_present", read_state) monkeypatch.setattr(cell_place.apg, "direct_refusals", direct) monkeypatch.setattr(cell_place.apg, "take_lock", take_lock) @@ -224,9 +235,9 @@ async def test_local_noop_insert_still_commits_durability_message(local): conn, state = local assert await invoke_place() == {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} - assert conn.committed and conn.terminated + assert conn.committed and conn.closed and not conn.terminated assert conn.calls[0:2] == ["lock", "directory"] - assert conn.calls[-1] == pg._DURABLE_MESSAGE + assert conn.calls[-2:] == [pg._DURABLE_MESSAGE, "close"] @pytest.mark.parametrize( @@ -246,7 +257,7 @@ async def test_local_invalid_identity_refuses_before_directory(local, change): state.update(change) with pytest.raises(cell_place.PlacementRefused): await invoke_place() - assert conn.calls == ["lock"] and conn.terminated + assert conn.calls == ["lock", "close"] and conn.closed and not conn.terminated @pytest.mark.parametrize("change", [{"state": "recording"}, {"first": "us-1"}, {"row": None}]) @@ -262,7 +273,7 @@ async def changed(*a): with pytest.raises(cell_place.PlacementRefused): await invoke_place() assert not any("INSERT" in sql for sql in conn.calls) - assert conn.terminated and not conn.committed + assert conn.closed and not conn.terminated and not conn.committed async def test_local_insert_failure_can_be_retried(local): @@ -270,7 +281,7 @@ async def test_local_insert_failure_can_be_retried(local): conn.fail_insert = True with pytest.raises(OSError, match="local insert failed"): await invoke_place() - assert not conn.committed and conn.terminated + assert not conn.committed and conn.closed and not conn.terminated conn.fail_insert = False await invoke_place() assert conn.committed @@ -295,7 +306,7 @@ async def test_local_commit_cancellation_closes_the_session(local): task.cancel() with pytest.raises(asyncio.CancelledError): await task - assert conn.terminated and not conn.committed + assert conn.closed and not conn.terminated and not conn.committed async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): @@ -304,16 +315,14 @@ async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): conn, state = local cleaned = False - @asynccontextmanager async def stalled(**kw): nonlocal cleaned try: await asyncio.Event().wait() - yield conn finally: cleaned = True - monkeypatch.setattr(pg, "dedicated", stalled) + monkeypatch.setattr(pg, "_connect_dedicated", stalled) with pytest.raises(TimeoutError): await asyncio.wait_for(invoke_place(), 0.5) assert cleaned and conn.calls == [] @@ -346,7 +355,7 @@ async def stalled(sql, *args): assert done, "the statement deadline must include ending its transaction" with pytest.raises(TimeoutError): await task - assert conn.terminated + assert conn.closed and not conn.terminated finally: if not task.done(): task.cancel() @@ -464,3 +473,122 @@ async def cancel(*args): printed = capsys.readouterr() assert printed.out == "" assert "placement may have committed; rerun the same command" in printed.err + + +async def test_local_phases_remain_separate_from_directory_and_exit(local, monkeypatch): + from server.cli import cell_control, cell_place + + observed = [] + actual_call = cell_control.Owner.call + + async def timed(self, work, *, total_s): + observed.append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def directory_call(*args): + # HTTP owns its existing independent transport limit, not a local phase cap. + await asyncio.sleep(0.06) + + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(cell_place, "_directory", directory_call) + await invoke_place() + assert observed == [0.03, 0.03, 0.03] + conn, _ = local + assert conn.committed and conn.closed and not conn.terminated + + +@pytest.mark.parametrize("where", ["initialization", "initial", "transaction", "directory"]) +async def test_original_error_survives_failed_required_close(local, monkeypatch, where): + from server.cli import cell_place + from server.core import pg + + original = ValueError("original placement failure") + conn, _ = local + + async def fail(*args): + raise original + + async def bad_close(**kwargs): + raise RuntimeError("close failed") + + conn.close = bad_close + if where == "initialization": + monkeypatch.setattr(pg, "init_connection", fail) + elif where == "initial": + monkeypatch.setattr(cell_place, "_identity", fail) + elif where == "directory": + monkeypatch.setattr(cell_place, "_directory", fail) + else: + conn.execute = fail + with pytest.raises(ValueError) as caught: + await invoke_place() + assert caught.value is original + assert conn.terminated and not conn.committed + + +async def test_success_refuses_when_required_close_fails(local): + conn, _ = local + + async def fail(**kwargs): + raise OSError("required close failed") + + conn.close = fail + with pytest.raises(OSError, match="required close failed"): + await invoke_place() + assert conn.committed and conn.terminated + + +async def test_lost_original_lock_refuses_without_reconnecting(local, monkeypatch): + from server.cli import cell_place + + conn, _ = local + + async def lost(c): + assert c is conn + return False + + monkeypatch.setattr(cell_place.apg, "lock_held", lost) + with pytest.raises(cell_place.PlacementRefused, match="lock was lost"): + await invoke_place() + assert conn.calls == ["lock", "directory", "close"] + assert conn.closed and not conn.committed + + +@pytest.mark.parametrize("inside_reserve", [False, True]) +async def test_late_placement_connection_never_reaches_body(local, monkeypatch, inside_reserve): + from server.core import pg + + conn, _ = local + released, discarded = asyncio.Event(), asyncio.Event() + real_terminate = conn.terminate + real_close = conn.close + + def terminate(): + real_terminate() + discarded.set() + + async def close(**kw): + await real_close(**kw) + discarded.set() + + async def late(**kwargs): + try: + await asyncio.Future() + except asyncio.CancelledError: + if not inside_reserve: + while not released.is_set(): + try: + await released.wait() + except asyncio.CancelledError: + pass + return conn + + conn.terminate, conn.close = terminate, close + monkeypatch.setattr(pg, "_connect_dedicated", late) + with pytest.raises(TimeoutError): + await invoke_place() + released.set() + await asyncio.wait_for(discarded.wait(), 0.2) + assert conn.calls == (["close"] if inside_reserve else []) + assert conn.closed if inside_reserve else conn.terminated + assert not conn.committed diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py new file mode 100644 index 0000000000..1b8cd967e7 --- /dev/null +++ b/tests/unit/test_cell_place_owner.py @@ -0,0 +1,307 @@ +"""Placement reuses the control owner without replacing its phase budgets.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control +from server.core import pg +from tests.unit.test_cell_attach import FakeConn + +pytestmark = pytest.mark.asyncio + + +async def test_custom_exit_is_used_only_outside_a_local_phase(): + owner = cell_control.Owner(exit_s=0.09) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + before = time.monotonic() + assert await owner.run(opening, AsyncMock(return_value=17), opening_s=0.3) == 17 + assert before + 0.09 <= owner.close_end <= time.monotonic() + 0.09 + assert conn.executed == ["CLOSE"] + + +@pytest.mark.parametrize("occupied", ["conn", "phase", "task", "closed", "repeated"]) +async def test_command_entry_refuses_reuse_before_any_callback(occupied): + owner = cell_control.Owner() + held = None + if occupied == "conn": + owner.conn = FakeConn() + elif occupied == "phase": + owner.phase_end = time.monotonic() + 1 + elif occupied == "task": + held = owner._start(lambda: asyncio.Future()) + elif occupied == "closed": + await owner.close() + else: + await owner.run(AsyncMock(), AsyncMock(), opening_s=0.1) + opening, body = AsyncMock(), AsyncMock() + try: + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(opening, body, opening_s=0.1) + opening.assert_not_called() + body.assert_not_called() + finally: + if held is not None: + held.cancel() + await asyncio.gather(held, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "body"]) +async def test_recursive_entry_cannot_replace_the_original_backend(stage): + owner = cell_control.Owner() + original = FakeConn() + other_open, other_body = AsyncMock(), AsyncMock() + + async def recurse(): + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(other_open, other_body, opening_s=0.1) + + async def opening(): + if stage == "opening": + await recurse() + owner.conn = original + + async def body(): + if stage == "body": + await recurse() + return 3 + + assert await owner.run(opening, body, opening_s=0.1) == 3 + other_open.assert_not_called() + other_body.assert_not_called() + assert owner.conn is original and original.executed == ["CLOSE"] + + +async def test_native_wrapper_keeps_its_existing_three_budgets(monkeypatch): + observed = {} + actual_run = cell_control.Owner.run + actual_call = cell_control.Owner.call + + async def wrapped_run(self, opening, work, *, opening_s): + observed.update(opening=opening_s, exit=self.exit_s) + return await actual_run(self, opening, work, opening_s=opening_s) + + async def wrapped_call(self, work, *, total_s): + observed.setdefault("phases", []).append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def opening(self, what): + assert what == "native" + self.conn = FakeConn() + + monkeypatch.setattr(cell_control.Owner, "run", wrapped_run) + monkeypatch.setattr(cell_control.Owner, "call", wrapped_call) + monkeypatch.setattr(cell_control.Owner, "_open", opening) + assert ( + await cell_control.run("native", lambda owner: owner.transaction(AsyncMock(return_value=7))) + == 7 + ) + assert observed == {"opening": 75.0, "exit": pg._RELEASE_TIMEOUT_S, "phases": [75.0, 60.0]} + + +async def test_failed_phase_keeps_its_end_instead_of_custom_exit(): + owner = cell_control.Owner(exit_s=99) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + async def body(): + return await owner.call(lambda: asyncio.Future(), total_s=0.04) + + before = time.monotonic() + with pytest.raises(TimeoutError): + await owner.run(opening, body, opening_s=0.1) + assert before < owner.close_end <= before + 0.1 + assert owner.close_end == owner.phase_end + assert conn.executed == ["CLOSE"] + + +async def test_public_close_defers_repeated_signals_until_acknowledged_within_original_end(): + events = [] + closing, finish = asyncio.Event(), asyncio.Event() + + class Connection: + async def close(self, **kwargs): + events.append("close") + closing.set() + await finish.wait() + events.append("ack") + + def terminate(self): + events.append("abort") + + def is_closed(self): + return "ack" in events + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + original_end = owner.close_end + task.cancel("first signal") + await asyncio.sleep(0) + task.cancel("second signal") + await asyncio.sleep(0) + assert not task.done(), "a second signal must not interrupt owned public close" + assert owner.close_end == original_end + finish.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert events == ["close", "ack"] + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "initial", "directory", "transaction", "close"]) +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +async def test_placement_fatal_stays_inside_owner_until_public_close(stage, kind): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse, asyncio, sys + import httpx + from server.cli import cell_place + from server.core import pg + stage, kind = sys.argv[1:] + original = (SystemExit if kind == 'SystemExit' else KeyboardInterrupt)('original') + events = [] + class Transaction: + async def start(self): pass + async def commit(self): events.append('commit') + class Connection: + def transaction(self): return Transaction() + async def execute(self, *args): + if stage == 'transaction': raise original + async def close(self, **kw): + events.append('close') + if stage == 'close': raise original + def terminate(self): events.append('abort') + conn = Connection() + async def connect(**kw): return conn + async def initialize(c): + assert c is conn + if stage == 'opening': raise original + async def noop(*args): pass + async def direct(c, what): return [] + async def identity(c, name): + assert c is conn + if stage == 'initial': raise original + return 'dep', {} + async def directory(*args): + if stage == 'directory': raise original + async def held(c): return True + async def main(): + pg._connect_dedicated = connect + pg._check_deployment = initialize + pg.init_connection = noop + cell_place.apg.direct_refusals = direct + cell_place.apg.take_lock = noop + cell_place.apg.lock_held = held + cell_place._identity = identity + cell_place._directory = directory + try: + await cell_place.place(argparse.Namespace(cell='first', org='acme'), httpx.URL('http://router')) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 1, events + await asyncio.sleep(0) + print('original-after-public-close') + return + raise AssertionError('fatal became success') + asyncio.run(main()) + """ + ) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, "-c", program, stage, kind], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-public-close" in result.stdout + + +async def test_repeated_signals_cannot_extend_an_expired_public_close(): + closing = asyncio.Event() + conn = FakeConn() + + async def stalled(**kwargs): + closing.set() + await asyncio.Future() + + conn.close = stalled + owner = cell_control.Owner(exit_s=0.04) + owner.conn = conn + started = time.monotonic() + task = asyncio.create_task(owner.close()) + await closing.wait() + end = owner.close_end + + async def signals(): + while not task.done(): + task.cancel("first signal") + await asyncio.sleep(0.001) + + signal_task = asyncio.create_task(signals()) + try: + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "repeated signals cannot restart the original cleanup allowance" + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert time.monotonic() - started < 0.2 + assert owner.close_end == end + assert "TERMINATE" in conn.executed + finally: + signal_task.cancel() + await asyncio.gather(signal_task, task, return_exceptions=True) + + +async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): + closing, acknowledged = asyncio.Event(), asyncio.Event() + events = [] + + class Connection: + async def close(self, **kwargs): + closing.set() + await acknowledged.wait() + events.append("ack") + + def is_closed(self): + # An aborted transport is closed even while its cancellation is pending. + return True + + def terminate(self): + events.append("abort") + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + task.cancel("stop") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() and events == [] + acknowledged.set() + with pytest.raises(asyncio.CancelledError, match="stop"): + await task + assert events == ["ack"] + finally: + acknowledged.set() + await asyncio.gather(task, return_exceptions=True) From 8018e30c02f902dd360e5589cfe659919d387900 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:12:50 +0000 Subject: [PATCH 067/181] Keep attach cleanup inside its owning deadline Close the restoration session before cancelling transaction rollback, and capture context-exit failures without replacing the first operation error. Describe the required timing workload separately from the small diagnostic measurements. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 24 ++-- docs/benchmarks/results.md | 56 +++++--- server/cli/cell_attach.py | 43 ++---- server/cli/cell_attach_pg.py | 53 ++++++- tests/unit/test_bench_cell_trigger_gate.py | 131 ++++++++++++++++- tests/unit/test_cell_attach_owned_exit.py | 160 +++++++++++++++++++++ 6 files changed, 403 insertions(+), 64 deletions(-) create mode 100644 tests/unit/test_cell_attach_owned_exit.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 7321583095..b1874e1fd9 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -403,10 +403,10 @@ def close(self) -> None: self.file.close() -async def _close_pg(conn: Any) -> None: +async def _close_pg(conn: Any, end: float | None = None) -> None: from server.core import pg - await pg.close_owned(conn, end=asyncio.get_running_loop().time() + IO_S) + await pg.close_owned(conn, end=time.monotonic() + IO_S if end is None else end) class ReadOnlyCH: @@ -1307,9 +1307,12 @@ async def _run( artifact.emit("provisional_statistics", **summary) return summary finally: + from server.cli import cell_attach_pg as apg + prior = sys.exception() cleanup = asyncio.create_task( - _cleanup(triggers, conn, artifact), name="trigger-gate-cleanup" + apg._work_result(lambda: _cleanup(triggers, conn, artifact)), + name="trigger-gate-cleanup", ) interrupted = False while not cleanup.done(): @@ -1320,7 +1323,8 @@ async def _run( except BaseException: # noqa: BLE001 — preserve an earlier operation failure break try: - cleanup.result() + if isinstance(result := cleanup.result(), apg._Interrupted): + raise result.error except BaseException as error: try: artifact.emit("cleanup_error", error_type=type(error).__name__) @@ -1334,23 +1338,25 @@ async def _run( async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: + end = time.monotonic() + RESTORE_S try: if triggers is not None: - await _restore(triggers, conn) + await _restore(triggers, conn, end) artifact.emit("triggers_restored", verified=True) finally: prior = sys.exception() try: - await _close_pg(conn) + await _close_pg(conn, end) except BaseException: if prior is None: raise artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) -async def _restore(triggers: Triggers, conn: Any) -> None: - async with asyncio.timeout(RESTORE_S): - await triggers.restore(conn) +async def _restore(triggers: Triggers, conn: Any, end: float) -> None: + from server.cli import cell_attach_pg as apg + + await apg.supervise(conn, triggers.restore(conn), end=end) def main() -> int: diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 4814bd7f5f..0f9bd094b1 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1222,17 +1222,11 @@ at most 10% slower) holds by a wide margin; the cost lands on scripts and older writing a cell without the check. This measures the triggers on the statements, not a whole `memory.set` through a server on a cell; the end-to-end gate follows. -`bench/cell_trigger_gate.py` measures the gate end to end, locally: a first cell attached -by `loom cell attach` (two server processes, a router, Postgres 16 with a synchronous -standby and ClickHouse in Docker on a shared host under other load), 400 sequential -`memory.set` calls of a placed org through one server, and a delivery-sized 1,000-row insert -of the held org from a marked and an unmarked session, the org triggers on and dropped in -turn, three rounds each; the harness checks the triggers it asked for are enabled: - -```bash -PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url \ - --pg-dsn --token --org --namespace -``` +A small local diagnostic on an attached first cell records 400 sequential `memory.set` +calls per run and 1,000-row statement inserts from marked and unmarked sessions, three +rounds with org triggers on and off. Its fixture has two server processes, a router, +Postgres 16 with a synchronous standby and ClickHouse in Docker on a shared host under +other load. These are diagnostic figures, not the end-to-end acceptance workload: | Local, three rounds | Triggers on | Triggers off | | ----------------------------------- | ---------------------- | ---------------------- | @@ -1241,17 +1235,35 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py --base-url | 1,000-row insert, marked (median) | 11.0, 9.2, 8.5 ms | 9.7, 11.1, 10.6 ms | | 1,000-row insert, unmarked (median) | 22.1, 21.4, 21.0 ms | 10.9, 9.6, 11.0 ms | -The marked batch is what a delivery pass of a cell's own processes writes. On minus off -within each round it moved +1.3, −1.9 and −2.1 ms (+13.4%, −17.1% and −19.8%); the means are -9.6 ms on and 10.5 ms off, −0.9 ms (−8.6%). The delivery-pass gate (at most 10% slower) is -met on the mean, but one round of three was 13% slower and three rounds give no paired bound -(methodology's paired bootstrap is for per-item labels), so this run does not establish the -gate on its own; the statement-level figures above do, at under a microsecond a row against -about 10 µs a row of insert. The `memory.set` figures move by more between rounds of one -configuration than any trigger cost: on this loaded host the p99 differences within a round -(+214.8, +47.0, +24.3 ms here, and −71.7, −10.0, −42.4 ms in an earlier run of the same -measurement) have no consistent sign: the `memory.set` p99 gate (a rise of at most 10 ms) is -unresolved locally. An unmarked session pays about 11 µs a row, as above. +The marked statement medians average 9.6 ms on and 10.5 ms off. A statement insert omits +the delivery worker's claim, embedding, publication and settlement work, so these figures +establish no delivery-throughput gate. The API p99 differences within each round are ++214.8, +47.0 and +24.3 ms; another small diagnostic has differences of −71.7, −10.0 and +−42.4 ms. Their opposing signs on the loaded host leave the API gate unresolved. Neither +end-to-end gate is established by these measurements. + +`bench/cell_trigger_gate.py` measures the acceptance workload on an independently owned, +attached scratch fixture with explicit fake providers, enforced mode and an already held +org. Each API run has 50 unmeasured warmups and 2,000 sequential measured requests with +4,096-byte values, with five runs per arm in counterbalanced order. Each delivery block +times five complete `pg_delivery.run_pass` calls at the unchanged configured batch size, +with five blocks per arm. The latency gate is `median(p99_on) - median(p99_off) <= 10 ms`; +the throughput gate is `1 - median(rows_per_second_on) / median(rows_per_second_off) <= 0.10`. +The driver retains every run and raw outcome; it does not substitute statement timings. + +```bash +PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \ + --base-url --pg-dsn --token \ + --org --namespace --fixture-manifest \ + --launch-manifest --output +``` + +The fixture owner supplies the exact source, launch, configuration and ownership receipts, +excludes competing controllers and workers, and stops its processes before deleting its +exact scratch databases. The output path must not exist. The driver verifies those +receipts and the live fixture before native setup and every arm, restores the org triggers +within its cleanup bound and emits a final verdict only after cleanup and artifact sync. +Source tests and the diagnostic figures above are not evidence that this workload passes. ## Reference computations diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 2c3915ae66..c56f7b25f4 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -196,30 +196,11 @@ async def _take_lock(self) -> None: raise Refused(refusals) await apg.take_lock(self.lock_conn, "attach") - async def close_lock(self) -> None: - from server.core import pg - + async def close_lock(self, error: BaseException | None = None) -> None: if self._lock_cm is not None: cm, conn = self._lock_cm, self.lock_conn self._lock_cm = self.lock_conn = None - closed = pg._lost(conn) - end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) - try: - if not closed: - await pg.close_owned(conn, end=end) - finally: - task = asyncio.create_task(cm.__aexit__(None, None, None)) - task.add_done_callback(pg._retrieve_task) - try: - done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) - if not done: - task.cancel() - raise TimeoutError("attach lock cleanup deadline expired") - task.result() - except BaseException: - pg._terminate(conn) - task.cancel() - raise + await apg.close_context(cm, conn, error=error) async def confirm_lock(self) -> None: """The lock is held still, or taken again (refusing when another run holds it now), @@ -485,16 +466,22 @@ def mark(mode: str) -> None: # landed; `recording` again when that try rolled back. self.mode = mode + cm = pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) + conn = await cm.__aenter__() + failure: BaseException | None = None try: - async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: - await apg.commit_enforced( - conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark - ) + await apg.commit_enforced( + conn, self.confirm_lock, self.ready_for_commit, self.wait, self.log, mark + ) + except BaseException as error: # noqa: BLE001 — exit preserves the work failure. + failure = error + try: + await apg.close_context(cm, conn, error=failure) except Refused: raise except Exception as e: - if self.mode != "unknown": - raise # no try was in flight: nothing to decide + if failure is None or self.mode != "unknown": + raise # Only interrupted work needs the unknown-outcome recovery. self.log(f"the mode commit's outcome is unknown ({type(e).__name__}: {e})") async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await apg.end_mode_transactions(conn, self.confirm_lock) @@ -691,7 +678,7 @@ async def run(self) -> int: raise finally: try: - await self.close_lock() + await self.close_lock(failure) except BaseException: if failure is None: raise diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index c71e1130c7..ca6fedf794 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -94,9 +94,9 @@ class _Interrupted: error: BaseException -async def _work_result[T](work: Coroutine[Any, Any, T]) -> T | _Interrupted: +async def _work_result[T](work: Callable[[], Awaitable[T]]) -> T | _Interrupted: try: - return await work + return await work() except (KeyboardInterrupt, SystemExit) as error: # These exceptions otherwise stop asyncio's loop before the owner can close. return _Interrupted(error) @@ -108,7 +108,7 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - A failed borrowed transaction must remain alive until public close sends its cancellation. Closing the transport first can leave its backend locks held. """ - task = asyncio.create_task(_work_result(work)) + task = asyncio.create_task(_work_result(lambda: work)) reserve = min(1.0, max(0.0, end - time.monotonic()) / 4) try: done, _ = await asyncio.wait({task}, timeout=max(0.0, end - reserve - time.monotonic())) @@ -133,6 +133,53 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - raise +async def close_context( + cm: contextlib.AbstractAsyncContextManager[Any], + conn: Any, + *, + error: BaseException | None = None, +) -> None: + """Close an owned session and its context without replacing its first failure. + + Work supervision may have closed it already; context exit then gets no fresh + allowance. Both exit invocation and awaiting run inside the fatal capture. + """ + closed = pg._lost(conn) + end = time.monotonic() + (0.0 if closed else pg._RELEASE_TIMEOUT_S) + failure = error + try: + if not closed: + await pg.close_owned(conn, end=end) + except BaseException as caught: # noqa: BLE001 — keep the first operation failure. + if failure is None: + failure = caught + task = asyncio.create_task( + _work_result( + lambda: cm.__aexit__( + type(failure) if failure is not None else None, + failure, + failure.__traceback__ if failure is not None else None, + ) + ) + ) + try: + done, _ = await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) + if not done: + raise TimeoutError("attach context cleanup deadline expired") + if isinstance(result := task.result(), _Interrupted): + raise result.error + except BaseException as caught: # noqa: BLE001 — exit cannot replace the first failure. + pg._terminate(conn) + if failure is None: + failure = caught + finally: + if not task.done(): + task.cancel() + task.add_done_callback(pg._retrieve_task) + if failure is not None: + raise failure + + def _split(key: int) -> str: """The `pg_locks` columns that hold a bigint advisory key, in this database.""" return ( diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 31986733a8..29dcce134b 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -724,7 +724,7 @@ async def restore(*_): restoring.set() await release.wait() - async def close(c): + async def close(c, *_): c.closed = True monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=conn)) @@ -992,7 +992,7 @@ async def restore(self, _): monkeypatch.setattr(gate, "RESTORE_S", 0.01) - async def close(owned): + async def close(owned, *_): assert owned is conn await conn.close() @@ -1456,3 +1456,130 @@ def forbidden(*_args, **_kwargs): verdict = json.loads(capsys.readouterr().out) assert verdict["verdict"] == "INVALID" and verdict["reason"] == "launch receipt hash" assert not (tmp_path / "output.jsonl").exists() + + +@pytest.mark.asyncio +async def test_restore_closes_before_installed_transaction_rollback_can_overrun(monkeypatch): + from asyncpg.transaction import Transaction + + events = [] + release, rolling_back = asyncio.Event(), asyncio.Event() + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + self.timeouts = [] + + def is_closed(self): + return self.closed + + def transaction(self): + return Transaction(self, None, False, False) + + async def execute(self, sql, *args): + if sql.startswith("DROP TRIGGER"): + raise ValueError("restoration DDL failed") + if sql.startswith("ROLLBACK"): + rolling_back.set() + try: + while not release.is_set(): + try: + await release.wait() + except asyncio.CancelledError: + continue + finally: + events.append("rollback ended") + + async def close(self, **kwargs): + self.timeouts.append(kwargs["timeout"]) + events.append("public close") + self.closed = True + release.set() + + def terminate(self): + self.closed = True + release.set() + + conn = Connection() + monkeypatch.setattr(gate, "RESTORE_S", 0.04) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + task = asyncio.create_task( + gate._cleanup(gate.Triggers(("memory_current",), "", ""), conn, artifact) + ) + try: + await asyncio.wait_for(rolling_back.wait(), 1) + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "restoration deadline waited for rollback before closing" + with pytest.raises(TimeoutError): + task.result() + assert conn.closed + assert events.index("public close") < events.index("rollback ended") + assert conn.timeouts and all(0 < bound <= 0.04 for bound in conn.timeouts) + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("kind", ["SystemExit", "KeyboardInterrupt"]) +@pytest.mark.parametrize("phase", ["restore", "close"]) +def test_driver_cleanup_native_failure_stays_inside_its_owner(kind, phase): + import subprocess + import sys + import textwrap + + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from types import SimpleNamespace + from unittest.mock import AsyncMock + from bench import cell_trigger_gate as gate + kind, phase = sys.argv[1:] + original = ValueError('original operation failure') + secondary = {'SystemExit': SystemExit, 'KeyboardInterrupt': KeyboardInterrupt}[kind]( + 'cleanup failed') + events = [] + class Connection: + async def close(self, **kwargs): + events.append('close') + if phase == 'close': raise secondary + def terminate(self): events.append('abort') + class Triggers: + tables = ('memory_current',) + invariant = org = 'hash' + async def set(self, *args): pass + async def restore(self, *args): + events.append('restore') + if phase == 'restore': raise secondary + async def main(): + gate.asyncpg.connect = AsyncMock(return_value=Connection()) + gate._proof = AsyncMock(return_value={}) + gate.Triggers.capture = AsyncMock(return_value=Triggers()) + gate._memory_set = AsyncMock(side_effect=original) + artifact = SimpleNamespace(emit=lambda *a, **k: None) + try: + await gate._run(argparse.Namespace(pg_dsn='unused'), None, artifact, object()) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events[0] == 'restore' and 'close' in events, events + await asyncio.sleep(0) + print('original-after-driver-cleanup') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, kind, phase], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-driver-cleanup" in result.stdout diff --git a/tests/unit/test_cell_attach_owned_exit.py b/tests/unit/test_cell_attach_owned_exit.py new file mode 100644 index 0000000000..9bdaedb76d --- /dev/null +++ b/tests/unit/test_cell_attach_owned_exit.py @@ -0,0 +1,160 @@ +"""Owned attach contexts finish before an error reaches the command boundary.""" + +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.mark.parametrize("owner", ["lock", "mode"]) +@pytest.mark.parametrize("secondary", ["RuntimeError", "SystemExit", "KeyboardInterrupt"]) +def test_actual_owned_context_keeps_first_failure(owner, secondary): + program = textwrap.dedent( + """ + import argparse + import asyncio + import sys + from unittest.mock import AsyncMock + import asyncpg + from server.cli.cell_attach import Run + from server.cli import cell_attach_pg as apg + from server.core import pg + from server.settings import settings + + owner, kind = sys.argv[1:] + original = ValueError('operation failed') + secondary = {'RuntimeError': RuntimeError, 'SystemExit': SystemExit, + 'KeyboardInterrupt': KeyboardInterrupt}[kind]('exit failed') + events = [] + class Conn: + def __init__(self): + self.closed = False + self.closes = 0 + def is_closed(self): + return self.closed + async def close(self, **kwargs): + self.closes += 1 + events.append('close') + self.closed = True + if self.closes == 2: + raise secondary + def terminate(self): + self.closed = True + events.append('abort') + + async def main(): + conn = Conn() + async def raw(*args, **kwargs): return conn + async def checked(*args): pass + async def fail(*args): raise original + asyncpg.connect = raw + pg._check_deployment = checked + pg.init_connection = checked + settings.pg_dsn = 'postgresql://unused/test' + router = AsyncMock() + router.call.return_value = {'state': 'attaching'} + run = Run(None, argparse.Namespace(cell='test', wait=0), router, lambda _: None) + run.row = {'deployment_id': 'test'} + run.mode = 'recording' + run.confirm_lock = AsyncMock() + run.ready_for_commit = AsyncMock(return_value=[]) + async def open_lock(): + run._lock_cm = pg.dedicated() + run.lock_conn = await run._lock_cm.__aenter__() + run.open_lock = open_lock + run.read_own_row = AsyncMock(side_effect=original) + apg._mode_transaction = fail + try: + await (run.run() if owner == 'lock' else run.commit_mode()) + except BaseException as error: + assert error is original, (type(error).__name__, events) + assert events.count('close') == 2, events + await asyncio.sleep(0) + print('original-after-owned-exit') + return + raise AssertionError('operation failure was swallowed') + asyncio.run(main()) + """ + ) + result = subprocess.run( + [sys.executable, "-c", program, owner, secondary], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "original-after-owned-exit" in result.stdout + + +@pytest.mark.asyncio +@pytest.mark.parametrize("factory_failure", [False, True]) +async def test_mode_failure_context_receives_the_actual_exception(monkeypatch, factory_failure): + import argparse + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli.cell_attach import Run + from server.core import pg + + original = ValueError("mode failed") + observed = [] + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + + class Context: + async def __aenter__(self): + return conn + + def __aexit__(self, kind, error, traceback): + observed.append((kind, error, traceback)) + if factory_failure: + raise RuntimeError("exit construction failed") + return AsyncMock(side_effect=RuntimeError("exit failed"))() + + monkeypatch.setattr(pg, "dedicated", lambda **_: Context()) + monkeypatch.setattr(apg, "commit_enforced", AsyncMock(side_effect=original)) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.row = {"deployment_id": "test"} + run.mode = "recording" + with pytest.raises(ValueError) as result: + await run.commit_mode() + assert result.value is original + assert len(observed) == 1 + assert observed[0][0] is ValueError and observed[0][1] is original + assert observed[0][2] is not None + + +@pytest.mark.asyncio +async def test_successful_mode_work_still_fails_when_required_close_fails(monkeypatch): + import argparse + from contextlib import asynccontextmanager + from unittest.mock import AsyncMock + + from server.cli import cell_attach_pg as apg + from server.cli.cell_attach import Run + from server.core import pg + + original = OSError("public close failed") + conn = AsyncMock() + conn.is_closed = lambda: False + conn.terminate = lambda: None + conn.close.side_effect = original + + @asynccontextmanager + async def dedicated(**_): + yield conn + + monkeypatch.setattr(pg, "dedicated", dedicated) + + async def committed(*args): + args[-1]("unknown") + + monkeypatch.setattr(apg, "commit_enforced", committed) + run = Run(None, argparse.Namespace(cell="test", wait=0), AsyncMock(), lambda _: None) + run.row = {"deployment_id": "test"} + with pytest.raises(OSError) as result: + await run.commit_mode() + assert result.value is original From 40be106acc5f9462abb835d9b857d64fc12ea198 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:16:20 +0000 Subject: [PATCH 068/181] Reserve cell configuration and configured orgs atomically Signed-off-by: zlareb1 --- server/router/app.py | 43 +- server/router/directory.py | 163 +++++++- server/router/fingerprints.py | 45 +++ .../integration/test_cell_update_directory.py | 369 ++++++++++++++++++ tests/unit/test_cell_control_census.py | 12 + tests/unit/test_cell_update_pair.py | 94 +++++ 6 files changed, 715 insertions(+), 11 deletions(-) create mode 100644 tests/integration/test_cell_update_directory.py create mode 100644 tests/unit/test_cell_update_pair.py diff --git a/server/router/app.py b/server/router/app.py index 52c9d3d157..8a50a36cff 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -21,6 +21,8 @@ - `GET /cells/v1/operator/cells/`: the cell's row as attach reads it; - `POST /cells/v1/operator/cells//attach/begin`, `.../attach/place`, `.../attach/enforced` and `.../attach/serving`: the first cell's attach, in that order; +- `POST /cells/v1/operator/cells//configuration`: reserve or narrow the cell's + configuration after store proof, with the claimed generation and exact observed pair; - `DELETE /cells/v1/operator/cells/`: `loom cell remove` removes the row of a cell that is `attaching` with no placement; - `GET /livez` and `GET /readyz` (the directory answers, read at most once a second). @@ -62,7 +64,7 @@ from server.core import cell as cell_state from server.core.cell_probe import PATH as PROBE_PATH -from server.router import directory, routing +from server.router import directory, fingerprints, routing from server.router.limits import ( ATTACH_BATCH, ATTACH_BEGIN_MAX_BYTES, @@ -686,6 +688,40 @@ async def claim(self, request: Request) -> JSONResponse: lambda: directory.claim(request.path_params["cell"], expected), _attach_answer ) + async def configuration(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if set(body) != {"action", "target_fp", "keys_fp", "nonces"}: + return _refused(400, "configuration requires action, target_fp, keys_fp and nonces") + action, target, keys_fp = body["action"], body["target_fp"], body["keys_fp"] + if action not in ("config", "finish", "revert") or not fingerprints.readable(target): + return _refused(400, "configuration requires a supported action and readable target_fp") + if not isinstance(keys_fp, str) or not cell_state.SHA256.fullmatch(keys_fp): + return _refused(400, "keys_fp is a SHA-256 fingerprint") + if not _nonces_shaped(body["nonces"]): + return _refused(400, _NONCES_REFUSAL) + name = request.path_params["cell"] + found = await _read_attaching(name) + if isinstance(found, JSONResponse): + return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) + if refused := await self._stores_check(found, body["nonces"]): + return refused + response = await _answered( + lambda: directory.update_configuration( + name, target, action, keys_fp=keys_fp, refusal=placement_refusal, expected=expected + ), + _attach_answer, + ) + if response.status_code in (200, 503): + self._table_at = None + return response + async def attach_begin(self, request: Request) -> JSONResponse: """`attach/begin`: once the probe listener echoes attach's nonces from the stores the row names and the upstream reports the row's deployment id, write when attach @@ -1470,6 +1506,11 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), + Route( + "/cells/v1/operator/cells/{cell}/configuration", + router.configuration, + methods=["POST"], + ), Route("/cells/v1/operator/cells/{cell}", router.remove, methods=["DELETE"]), Route( "/cells/v1/operator/cells/{cell}/attach/begin", diff --git a/server/router/directory.py b/server/router/directory.py index 585e32c271..eaa6325b72 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -678,9 +678,8 @@ async def _place_one( and audit row only when the insert wrote it.""" if reason := refusal(candidate): raise PlacementRefused(reason) - quota = candidate.placements_max - if quota is not None and candidate.placed >= quota: - raise PlacementRefused(f"cell {cell_name!r} holds its quota of orgs") + if reason := _quota_refusal(candidate, 1): + raise PlacementRefused(reason) inserted = await conn.fetchval( "INSERT INTO loom_router.placements (org, cell) VALUES ($1, $2) " "ON CONFLICT DO NOTHING RETURNING org", @@ -699,6 +698,12 @@ async def _place_one( ) +def _quota_refusal(row: Cell, additional: int) -> str: + if row.placements_max is not None and row.placed + additional > row.placements_max: + return f"cell {row.cell!r} holds its quota of orgs" + return "" + + def _refuse_long(orgs: Iterable[str]) -> None: """OrgTooLong for the first org longer than `cell.ORG_MAX_BYTES`: the one check every route that inserts a placement runs, so no route places such an org.""" @@ -808,13 +813,10 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: "bucket_prefix": found.bucket_prefix, "host": row.host, } - for name, value in (("current_fp", begin.current_fp), ("next_fp", begin.next_fp)): - if value is None: - continue - if reason := fingerprints.registration_refusal(value, registered, name): - return reason - if reason := fingerprints.placement_refusal(row.cell, name, value): - return reason + if reason := fingerprints.registration_pair_refusal( + registered, begin.current_fp, begin.next_fp + ): + return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun # writes the fingerprints again only when they are the row's to the type. if row.current_fp is not None and _as_json(row.current_fp, row.next_fp) != _as_json( @@ -1031,6 +1033,147 @@ async def change_locked(conn: Any, _found: Attaching) -> Attaching: return await _attach_change(cell_name, expected, change_locked, claim=True) +async def update_configuration( + cell_name: str, + target: dict[str, Any], + action: str, + *, + keys_fp: str, + refusal: Callable[[Cell], str], + expected: Control, +) -> Attaching: + """Reserve acceptance and configured placements in one bounded directory commit.""" + try: + current, pending = fingerprints.configuration_pair( + expected.values.get("current_fp"), expected.values.get("next_fp"), target, action + ) + except ValueError as error: + raise OperatorRefused(str(error)) from None + accepted = [value for value in (current, pending) if value is not None] + desired_orgs = {org for value in accepted for org in value["fields"].get("configured_orgs", [])} + prior_orgs = { + org + for value in (expected.values["current_fp"], expected.values["next_fp"]) + if value is not None + for org in value["fields"].get("configured_orgs", []) + } + orgs = sorted(desired_orgs | prior_orgs) + _refuse_long(orgs) + + async def locked(conn: Any) -> Attaching: + # Numeric hash order matches live placement locks, including hash collisions. + await conn.execute( + "WITH keys AS MATERIALIZED (SELECT DISTINCT hashtext(org) AS key " + "FROM unnest($2::text[]) AS org ORDER BY key) " + "SELECT pg_advisory_xact_lock($1, key) FROM keys ORDER BY key", + _PLACE_LOCK, + orgs, + ) + await conn.execute("SELECT pg_advisory_xact_lock($1)", _DIRECTORY_LOCK) + raw = await conn.fetchrow( + f"SELECT {_CELL_COLUMNS}, {', '.join(_ATTACH_EXTRA)}, keys_fp " + "FROM loom_router.cells WHERE cell=$1 FOR UPDATE", + cell_name, + ) + if raw is None: + raise NoSuchCell(f"cell {cell_name!r} is not in the directory") + row = dict(raw) + recorded_keys = row.pop("keys_fp") + others = await conn.fetch( + "SELECT cell, current_fp, next_fp FROM loom_router.cells WHERE cell<>$1 ORDER BY cell", + cell_name, + ) + found = _attaching(row, str(others[0]["cell"]) if others else None) + if reason := control_refusal(found, expected): + raise OperatorRefused(reason) + if recorded_keys != keys_fp: + raise OperatorRefused( + "cell root-secret fingerprint changed; configuration update refuses" + ) + registered = {name: getattr(found, name) for name in ("bucket", "bucket_prefix")} + registered.update( + cell=cell_name, deployment_id=found.row.deployment_id, host=found.row.host + ) + if reason := fingerprints.registration_pair_refusal(registered, current, pending): + raise OperatorRefused(reason) + for other in others: + reserved = [ + other[name] for name in ("current_fp", "next_fp") if other[name] is not None + ] + if reason := fingerprints.reservation_refusal(accepted, reserved): + raise OperatorRefused(f"cell {other['cell']!r}: {reason}") + await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") + placed = await conn.fetch( + "SELECT org, cell FROM loom_router.placements WHERE org=ANY($1::text[]) ORDER BY org", + orgs, + ) + elsewhere = next((row for row in placed if row["cell"] != cell_name), None) + if elsewhere is not None: + raise OperatorRefused( + f"org {elsewhere['org']!r} is placed in cell {elsewhere['cell']!r}" + ) + new = sorted(set(orgs) - {row["org"] for row in placed}) + if not found.row.is_first and found.row.state != "serving": + if desired_orgs != prior_orgs: + raise OperatorRefused("a later cell must be serving before configured orgs change") + if set(new) & prior_orgs: + if not (found.row.is_first and found.row.state == "attaching"): + raise OperatorRefused( + "configured org placement is missing; " + "restore the directory metadata before updating" + ) + # Original first-cell orgs retain attach's placement and quota protocol. + new = sorted(set(new) - prior_orgs) + if new: + if reason := refusal(found.row): + raise OperatorRefused(reason) + if reason := _quota_refusal(found.row, len(new)): + raise OperatorRefused(reason) + return await _configuration_commit(conn, found, current, pending, new, action) + + return await _control_transaction(locked) + + +async def _configuration_commit( + conn: Any, + found: Attaching, + current: dict[str, Any], + pending: dict[str, Any] | None, + new_orgs: list[str], + action: str, +) -> Attaching: + accepted_issuers = sorted( + { + issuer + for value in (current, pending) + if value is not None + for issuer in value["fields"].get("bearer_issuers", []) + if issuer + } + ) + written = await conn.fetchrow( + "WITH inserted AS (INSERT INTO loom_router.placements (org, cell) " + "SELECT org, $1 FROM unnest($2::text[]) AS org RETURNING org), " + "placed_audit AS (INSERT INTO loom_router.audit (actor, action, detail) " + "SELECT 'operator', 'place', jsonb_build_object('org', org, 'cell', $1::text) " + "FROM inserted), " + "changed AS (UPDATE loom_router.cells SET current_fp=$3, next_fp=$4, " + "issuers=$5, placed=placed+(SELECT count(*) FROM inserted), updated_at=now() " + f"WHERE cell=$1 RETURNING {_CELL_COLUMNS}, {', '.join(_ATTACH_EXTRA)}), " + "changed_audit AS (INSERT INTO loom_router.audit (actor, action, detail) " + "SELECT 'operator', 'configuration_' || $6::text, " + "jsonb_build_object('cell', cell, 'placed', (SELECT count(*) FROM inserted)) FROM changed) " + "SELECT * FROM changed", + found.row.cell, + new_orgs, + current, + pending, + accepted_issuers if found.row.state == "serving" else [], + action, + ) + return _attaching(written, found.other) + + async def _audit(conn: Any, action: str, detail: dict[str, Any]) -> None: await conn.execute( "INSERT INTO loom_router.audit (actor, action, detail) VALUES ('operator', $1, $2)", diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py index 8b30894443..884948b609 100644 --- a/server/router/fingerprints.py +++ b/server/router/fingerprints.py @@ -82,6 +82,19 @@ def registration_refusal(value: Any, registered: Mapping[str, Any], name: str = return "" +def registration_pair_refusal( + registered: Mapping[str, Any], current: dict[str, Any], pending: dict[str, Any] | None +) -> str: + """Both reserved values must name the registered stores and fit live placement.""" + for name, value in (("current_fp", current), ("next_fp", pending)): + if value is not None: + if reason := registration_refusal(value, registered, name): + return reason + if reason := placement_refusal(registered["cell"], name, value): + return reason + return "" + + def _pair_refusal(a: dict[str, Any], b: dict[str, Any]) -> str: if set(a.get("bearer_issuers", [])) & set(b.get("bearer_issuers", [])): return "shared bearer issuer reservation" @@ -117,6 +130,38 @@ def reservation_refusal(accepted: Sequence[dict[str, Any]], other: Sequence[dict return "" +def configuration_pair( + current: dict[str, Any] | None, + pending: dict[str, Any] | None, + target: dict[str, Any], + action: str, +) -> tuple[dict[str, Any], dict[str, Any] | None]: + """Select the reserved pair without guessing the direction of an interrupted update.""" + if any(not readable(value) for value in (current, target)) or ( + pending is not None and not readable(pending) + ): + raise ValueError("configuration update needs readable fingerprints") + assert current is not None + if action not in ("config", "finish", "revert"): + raise ValueError("configuration action must be config, finish or revert") + if target["fields"].get("scim", cell.DEFAULTS["scim"]): + raise ValueError("configuration update requires per-cell SCIM before enabling SCIM") + + def same(a: dict[str, Any], b: dict[str, Any]) -> bool: + return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + if action == "config": + if pending is not None and not same(pending, target): + raise ValueError("another configuration is pending; finish or revert it first") + if pending is not None: + return current, pending + return current, None if same(current, target) else target + selected = pending if action == "finish" and pending is not None else current + if not same(selected, target): + raise ValueError("configuration target differs from the selected reserved fingerprint") + return target, None + + # An org as a placement body carries it at most: ORG_MAX_BYTES bytes, each escaped in JSON # to at most six (`\u00XX`). _PLACEMENT_ORG_MAX = 6 * cell.ORG_MAX_BYTES # not a setting: derived diff --git a/tests/integration/test_cell_update_directory.py b/tests/integration/test_cell_update_directory.py new file mode 100644 index 0000000000..3a2d95007c --- /dev/null +++ b/tests/integration/test_cell_update_directory.py @@ -0,0 +1,369 @@ +"""Configuration reservations, issuer routing and configured placements commit together.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from contextlib import asynccontextmanager +from typing import Any + +import httpx +import pytest + +from server.core import cell +from server.router import app, directory +from server.settings import settings +from tests.integration.test_router_directory_pg import dsn as dsn + + +def fingerprint(name: str, issuer: str, orgs: tuple[str, ...] = ()) -> dict[str, Any]: + return cell.as_value( + { + "deployment_id": f"dep-{name}", + "public_host": f"{name}.example.test", + "oauth_issuer_host": f"{name}.example.test", + "bearer_issuers": [issuer], + "configured_orgs": list(orgs), + } + ) + + +async def observed(name: str, *, claim: bool = True) -> directory.Control: + found = await directory.attach_read(name) + expected = directory.Control( + { + key: getattr(found.row, key) if hasattr(found.row, key) else getattr(found, key) + for key in directory.CONTROL_FIELDS + }, + found.row.mutation_generation, + ) + if claim: + found = await directory.claim(name, directory.Control(expected.values, None)) + expected = directory.Control(expected.values, found.row.mutation_generation) + return expected + + +@pytest.fixture +async def configured(dsn: str): + import asyncpg + + await directory.migrate() + conn = await asyncpg.connect(dsn) + try: + for name in ("first", "other"): + await conn.execute( + "INSERT INTO loom_router.cells " + "(cell, is_first, host, state, mode, credential_sha256, deployment_id, " + "current_fp, keys_fp, issuers, attach_started_at) " + "VALUES ($1, $2, $3, 'serving', 'enforced', $4, $5, $6::jsonb, $7, $8, now())", + name, + name == "first", + f"{name}.example.test", + ("a" if name == "first" else "b") * 64, + f"dep-{name}", + json.dumps(fingerprint(name, name)), + "c" * 64, + [name], + ) + yield conn + finally: + await conn.close() + + +async def change( + target: dict[str, Any], action: str = "config", *, expected: directory.Control | None = None +): + return await directory.update_configuration( + "first", + target, + action, + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=expected or await observed("first"), + ) + + +async def stored(conn: Any) -> dict[str, Any]: + row = await conn.fetchrow( + "SELECT current_fp, next_fp, issuers, placed, state, mode FROM loom_router.cells WHERE cell='first'" + ) + return dict(row) + + +@pytest.mark.asyncio +async def test_update_reserves_both_issuers_and_places_configured_orgs_once( + configured: Any, +) -> None: + target = fingerprint("first", "new", ("configured-a", "configured-b")) + await change(target) + row = await stored(configured) + assert json.loads(row["current_fp"]) == fingerprint("first", "first") + assert json.loads(row["next_fp"]) == target + assert row["issuers"] == ["first", "new"] + assert row["placed"] == 2 + assert (row["state"], row["mode"]) == ("serving", "enforced") + await configured.execute( + "UPDATE loom_router.cells SET closed=true, placements_max=2 WHERE cell='first'" + ) + await change(target) + assert await stored(configured) == row + assert ( + await configured.fetchval("SELECT count(*) FROM loom_router.audit WHERE action='place'") + == 2 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("blocked", ["closed", "quota", "other-cell", "overlap", "root-keys"]) +async def test_update_refusal_leaves_no_partial_reservation_or_placement( + configured: Any, blocked: str +) -> None: + target = fingerprint("first", "new", ("a-new", "z-blocked")) + if blocked == "closed": + await configured.execute("UPDATE loom_router.cells SET closed=true WHERE cell='first'") + elif blocked == "quota": + await configured.execute("UPDATE loom_router.cells SET placements_max=1 WHERE cell='first'") + elif blocked == "other-cell": + await configured.execute( + "INSERT INTO loom_router.placements (org,cell) VALUES ('z-blocked','other')" + ) + elif blocked == "overlap": + target["fields"]["bearer_issuers"] = ["other"] + else: + await configured.execute( + "UPDATE loom_router.cells SET keys_fp=$1 WHERE cell='first'", "d" * 64 + ) + before = await stored(configured) + with pytest.raises((directory.OperatorRefused, directory.PlacementRefused)): + await change(target) + assert await stored(configured) == before + assert ( + await configured.fetchval("SELECT count(*) FROM loom_router.placements WHERE cell='first'") + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action,target_issuer", [("finish", "new"), ("revert", "first")]) +async def test_update_narrowing_retains_permanent_placements( + configured: Any, action: str, target_issuer: str +) -> None: + pending = fingerprint("first", "new", ("permanent",)) + await change(pending) + target = pending if action == "finish" else fingerprint("first", "first") + await change(target, action) + row = await stored(configured) + assert json.loads(row["current_fp"]) == target + assert row["next_fp"] is None + assert row["issuers"] == [target_issuer] + assert row["placed"] == 1 + assert ( + await configured.fetchval("SELECT cell FROM loom_router.placements WHERE org='permanent'") + == "first" + ) + await change(target, action) + assert await stored(configured) == row + + +@pytest.mark.asyncio +async def test_update_old_generation_cannot_narrow_after_another_claim(configured: Any) -> None: + target = fingerprint("first", "new") + await change(target) + stale = await observed("first") + await observed("first") + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="generation"): + await change(target, "finish", expected=stale) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_concurrent_cells_cannot_reserve_one_issuer(configured: Any) -> None: + first, other = await observed("first"), await observed("other") + results = await asyncio.wait_for( + asyncio.gather( + change(fingerprint("first", "contested", ("first-org",)), expected=first), + directory.update_configuration( + "other", + fingerprint("other", "contested", ("other-org",)), + "config", + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=other, + ), + return_exceptions=True, + ), + timeout=15, + ) + assert sum(isinstance(result, directory.Attaching) for result in results) == 1 + assert sum(isinstance(result, directory.OperatorRefused) for result in results) == 1 + assert await configured.fetchval("SELECT sum(placed) FROM loom_router.cells") == 1 + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + + +@pytest.mark.asyncio +async def test_unreadable_other_reservation_cannot_be_replaced(configured: Any) -> None: + await configured.execute( + "UPDATE loom_router.cells SET next_fp=$1::jsonb WHERE cell='other'", '{"v":2,"fields":{}}' + ) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="unreadable"): + await change(fingerprint("first", "new", ("unplaced",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_attaching_cell_keeps_issuers_unroutable(configured: Any) -> None: + await configured.execute( + "UPDATE loom_router.cells SET state='attaching', issuers='{}' WHERE cell='first'" + ) + await change(fingerprint("first", "new", ("configured",))) + row = await stored(configured) + assert row["issuers"] == [] + assert row["state"] == "attaching" + assert row["placed"] == 1 + + +@pytest.mark.asyncio +async def test_later_attaching_cell_refuses_configured_org_changes(configured: Any) -> None: + await configured.execute("UPDATE loom_router.cells SET state='attaching' WHERE cell='other'") + with pytest.raises(directory.OperatorRefused, match="later cell"): + await directory.update_configuration( + "other", + fingerprint("other", "new", ("unplaced",)), + "config", + keys_fp="c" * 64, + refusal=app.placement_refusal, + expected=await observed("other"), + ) + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert ( + await configured.fetchval("SELECT next_fp FROM loom_router.cells WHERE cell='other'") + is None + ) + + +@pytest.mark.asyncio +async def test_update_cannot_replace_another_pending_value(configured: Any) -> None: + await change(fingerprint("first", "new")) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="pending"): + await change(fingerprint("first", "third", ("unplaced",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_recording_attach_leaves_original_configured_placement_to_attach( + configured: Any, +) -> None: + original = fingerprint("first", "first", ("served-before-attach",)) + await configured.execute( + "UPDATE loom_router.cells SET state='attaching', mode='recording', current_fp=$1::jsonb, issuers='{}' WHERE cell='first'", + json.dumps(original), + ) + await change(fingerprint("first", "new", ("served-before-attach",))) + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert (await stored(configured))["placed"] == 0 + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="pending"): + await change(fingerprint("first", "third", ("served-before-attach", "new-org"))) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_update_does_not_rebuild_missing_serving_directory_placement(configured: Any) -> None: + original = fingerprint("first", "first", ("missing-placement",)) + await configured.execute( + "UPDATE loom_router.cells SET current_fp=$1::jsonb WHERE cell='first'", json.dumps(original) + ) + before = await stored(configured) + with pytest.raises(directory.OperatorRefused, match="placement.*missing"): + await change(fingerprint("first", "new", ("missing-placement",))) + assert await stored(configured) == before + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + + +@pytest.mark.asyncio +async def test_configuration_fits_the_existing_operator_statement_budget( + configured: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + expected = await observed("first") + original = directory._connection + statements = [] + + @asynccontextmanager + async def counted(): + async with original() as conn: + conn.add_query_logger(lambda record: statements.append(record.query)) + yield conn + + monkeypatch.setattr(directory, "_connection", counted) + await change( + fingerprint("first", "new", tuple(f"configured-{i}" for i in range(40))), expected=expected + ) + await asyncio.sleep(0) # asyncpg schedules completed-query observers on the loop. + assert len(statements) == 9, statements + assert directory._bound_s(len(statements)) <= directory.CONTROL_MAX_S + assert await configured.fetchval("SELECT count(*) FROM loom_router.placements") == 40 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proof", ["ok", "refused", "new-claim"]) +async def test_configuration_http_requires_store_proof_before_any_change( + configured: Any, + monkeypatch: pytest.MonkeyPatch, + proof: str, +) -> None: + from starlette.responses import JSONResponse + + token = "lco_configuration-test" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + checked = [] + + async def stores_check(self, found, nonces): + checked.append((found.row.cell, nonces)) + if proof == "new-claim": + await observed("first") + return ( + JSONResponse({"error": "proof refused"}, status_code=409) + if proof == "refused" + else None + ) + + monkeypatch.setattr(app.Router, "_stores_check", stores_check) + expected = await observed("first") + target = fingerprint("first", "new", ("http-org",)) + nonces = {"postgres": "pg-nonce", "clickhouse": "ch-nonce", "bucket": None} + before = await stored(configured) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app()), base_url="http://router" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/configuration", + headers={"Authorization": f"Bearer {token}"}, + json={ + "action": "config", + "target_fp": target, + "keys_fp": "c" * 64, + "nonces": nonces, + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + }, + ) + assert response.status_code == (200 if proof == "ok" else 409), response.text + assert checked == [("first", nonces)] + if proof == "ok": + assert response.json()["next_fp"] == target + assert ( + await configured.fetchval( + "SELECT cell FROM loom_router.placements WHERE org='http-org'" + ) + == "first" + ) + else: + assert await stored(configured) == before diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 45b8e85baa..b77cb3f033 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -117,6 +117,9 @@ def test_cell_control_migrations_are_classified(): ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, + ("_configuration_commit", "INSERT INTO", "loom_router.placements"): 1, + ("_configuration_commit", "INSERT INTO", "loom_router.audit"): 2, + ("_configuration_commit", "UPDATE", "loom_router.cells"): 1, ("attach_begin.change_locked", "UPDATE", "loom_router.cells"): 1, ("attach_place.change_locked", "INSERT INTO", "loom_router.placements"): 1, ("attach_place.change_locked", "UPDATE", "loom_router.cells"): 1, @@ -436,6 +439,15 @@ def function(path: str, name: str) -> ast.AST: "claim", ["_attach_change(cell_name, expected, change_locked, claim=True)"], ), + ( + "server/router/directory.py", + "update_configuration", + [ + "_control_transaction(locked)", + "control_refusal(found, expected)", + "_configuration_commit(conn, found, current, pending, new, action)", + ], + ), *[ ( "server/router/directory.py", diff --git a/tests/unit/test_cell_update_pair.py b/tests/unit/test_cell_update_pair.py new file mode 100644 index 0000000000..6e12a1a0ba --- /dev/null +++ b/tests/unit/test_cell_update_pair.py @@ -0,0 +1,94 @@ +"""A configuration action reserves only its selected pair and preserves unknown fields.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +import pytest + +from server.core import cell +from server.router import fingerprints + + +def value(issuer: str) -> dict[str, Any]: + return cell.as_value({"bearer_issuers": [issuer], "future": {"nested": [False, 0]}}) + + +@pytest.mark.parametrize( + "action,pending,target,want", + [ + ("config", None, "new", ("old", "new")), + ("config", "new", "new", ("old", "new")), + ("config", None, "old", ("old", None)), + ("finish", "new", "new", ("new", None)), + ("revert", "new", "old", ("old", None)), + ("finish", None, "old", ("old", None)), + ("revert", None, "old", ("old", None)), + ], +) +def test_configuration_pair_selects_exact_reserved_values( + action: str, pending: str | None, target: str, want: tuple[str, str | None] +) -> None: + current, next_fp, chosen = value("old"), value(pending) if pending else None, value(target) + before = deepcopy((current, next_fp, chosen)) + result = fingerprints.configuration_pair(current, next_fp, chosen, action) + assert result == (value(want[0]), value(want[1]) if want[1] else None) + assert (current, next_fp, chosen) == before + + +@pytest.mark.parametrize( + "action,pending,target", + [ + ("config", "new", "third"), + ("config", "new", "old"), + ("finish", "new", "old"), + ("revert", "new", "new"), + ("finish", None, "new"), + ("revert", None, "new"), + ("unsupported", "new", "new"), + ], +) +def test_configuration_pair_refuses_a_different_pending_or_selected_target( + action: str, pending: str | None, target: str +) -> None: + with pytest.raises(ValueError): + fingerprints.configuration_pair( + value("old"), value(pending) if pending else None, value(target), action + ) + + +@pytest.mark.parametrize("broken", [None, {"v": True, "fields": {}}, {"v": 2, "fields": {}}]) +def test_configuration_pair_never_invents_an_unreadable_current(broken: Any) -> None: + with pytest.raises(ValueError, match="fingerprint"): + fingerprints.configuration_pair(broken, None, value("new"), "config") + + +def test_configuration_pair_uses_type_sensitive_unknown_field_equality() -> None: + current, pending, target = value("old"), value("new"), value("new") + target["fields"]["future"]["nested"][0] = 0 + assert pending == target # Python equality cannot implement the reservation CAS. + with pytest.raises(ValueError, match="pending"): + fingerprints.configuration_pair(current, pending, target, "config") + + +@pytest.mark.parametrize("action", ["config", "finish", "revert"]) +def test_configuration_pair_refuses_scim_without_enforcement(action: str) -> None: + target = value("new") + target["fields"]["scim"] = True + with pytest.raises(ValueError, match="per-cell SCIM"): + fingerprints.configuration_pair( + target if action == "revert" else value("old"), + target if action == "finish" else None, + target, + action, + ) + + +def test_configuration_pair_allows_revert_from_an_existing_scim_candidate() -> None: + pending = value("new") + pending["fields"]["scim"] = True + assert fingerprints.configuration_pair(value("old"), pending, value("old"), "revert") == ( + value("old"), + None, + ) From 8335af40b8ffa89aa063404c27d3757951a21ee6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:20:55 +0000 Subject: [PATCH 069/181] Describe configuration placement concurrency Signed-off-by: zlareb1 --- server/router/directory.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/router/directory.py b/server/router/directory.py index eaa6325b72..721f72d3d4 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -8,11 +8,12 @@ (`attach_place`) both take before they insert, so a cell's count and quota are judged one placement at a time; a live placement reads the winner again once it holds the lock, so an org a batch placed meanwhile is answered as placed; -- a transaction-scoped lock on the org (`_PLACE_LOCK`) between live placements, so two cells - placing one org at once agree on the winner, and a retry of the same call answers the - same; a batch takes none, since no other cell exists while it runs; -- the placements' primary key, as a backstop: every insert is `ON CONFLICT DO NOTHING`, and - only a row it returns is counted and audited. +- transaction-scoped org locks (`_PLACE_LOCK`) before cell locks for live placements and + configuration updates. Configuration takes distinct numeric hash keys in order; the + first attach batch needs none while no other cell exists; +- the placements' primary key, as a backstop. Live placement and first attach use + `ON CONFLICT DO NOTHING`; configuration inserts after its locked winner read. Only + inserted rows are counted and audited. """ from __future__ import annotations From c66786efc4aa06bd600df19e39deb02e09561b20 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:33:54 +0000 Subject: [PATCH 070/181] Preserve native owner errors when fallback abort fails Signed-off-by: zlareb1 --- server/cli/cell_control.py | 15 ++- tests/unit/test_cell_control_abort.py | 185 ++++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_cell_control_abort.py diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 74d804fdb6..1966b9e62b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -179,9 +179,13 @@ async def close(self) -> None: if isinstance(result, _Fatal): raise _ForwardFatal(result) except BaseException: - pg._terminate(self.conn) - if not self._close_task.done(): - self._close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first close failure. + pass + finally: + if not self._close_task.done(): + self._close_task.cancel() if interrupted is not None: raise interrupted from None raise @@ -197,7 +201,10 @@ async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. if self.conn is not None: - pg._terminate(self.conn) + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — failed abort preserves the caller's failure. + pass async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: if self.closed or self.phase_end is not None: diff --git a/tests/unit/test_cell_control_abort.py b/tests/unit/test_cell_control_abort.py new file mode 100644 index 0000000000..890ddcba27 --- /dev/null +++ b/tests/unit/test_cell_control_abort.py @@ -0,0 +1,185 @@ +"""A failed fallback abort cannot replace the native owner's first failure.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control + +pytestmark = pytest.mark.asyncio +ERRORS = (ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) +ABORT_ERRORS = (RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) + + +async def outcome(work): + try: + result = await cell_control._captured(work) + except BaseException as error: # noqa: BLE001 — observe the identical native failure inertly. + return error + assert isinstance(result, cell_control._Fatal), "a failed required close cannot succeed" + return result.error + + +@pytest.mark.parametrize("first_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_direct_owner_keeps_first_close_failure_and_retrieves_child( + monkeypatch, first_kind, abort_kind +): + first, secondary = first_kind("first close failure"), abort_kind("abort failure") + aborted = [] + + class Connection: + def terminate(self): + aborted.append(True) + raise secondary + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + close = AsyncMock(side_effect=first) + monkeypatch.setattr(cell_control.pg, "close_owned", close) + result = await outcome(owner.close) + assert result is first + assert aborted == [True] + close.assert_awaited_once() + assert owner.closed and owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +@pytest.mark.parametrize("first_kind", ERRORS) +async def test_outer_signal_wins_over_later_close_and_abort_failures( + monkeypatch, first_kind, abort_kind +): + entered, finish = asyncio.Event(), asyncio.Event() + first, secondary = first_kind("close failed after signal"), abort_kind("abort failure") + seen = [] + actual_wait = asyncio.wait + owner = cell_control.Owner(exit_s=0.2) + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + await finish.wait() + raise first + + async def wait(*args, **kwargs): + try: + return await actual_wait(*args, **kwargs) + except asyncio.CancelledError as error: + seen.append(error) + raise + + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + monkeypatch.setattr(cell_control.asyncio, "wait", wait) + task = asyncio.create_task(outcome(owner.close)) + try: + await entered.wait() + original_end = owner.close_end + task.cancel("original outer cancellation") + await asyncio.sleep(0) + assert len(seen) == 1 and not task.done() + finish.set() + result = await task + assert result is seen[0] + assert owner.close_end == original_end + assert owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_abort_failure_still_cancels_and_retrieves_expired_close_child( + monkeypatch, abort_kind +): + entered, ended = asyncio.Event(), asyncio.Event() + secondary = abort_kind("abort failure") + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + try: + await asyncio.Future() + finally: + ended.set() + raise ValueError("late close failure") + + owner = cell_control.Owner(exit_s=0.02) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert isinstance(result, TimeoutError) + assert time.monotonic() - started < 0.2 + assert entered.is_set() + await asyncio.wait_for(ended.wait(), 0.2) + await asyncio.sleep(0) + assert owner._close_task.done() and not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_self_cancelled_close_is_terminal_even_when_abort_fails(monkeypatch, abort_kind): + first = asyncio.CancelledError("close cancelled itself") + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failure") + + async def close(conn, *, end): + calls.append("close") + raise first + + owner = cell_control.Owner(exit_s=60) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert result is first + assert time.monotonic() - started < 0.2 + assert calls == ["close", "abort"] + + +@pytest.mark.parametrize("wait_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_failed_reap_abort_cannot_replace_the_callers_first_failure( + monkeypatch, wait_kind, abort_kind +): + owner = cell_control.Owner() + owner.close_end = time.monotonic() + 0.2 + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failed while reaping") + + async def failed_wait(*args, **kwargs): + raise wait_kind("reap wait interrupted") + + owner.conn = Connection() + child = owner._start(lambda: asyncio.Future()) + monkeypatch.setattr(cell_control.asyncio, "wait", failed_wait) + try: + try: + result = await cell_control._captured(lambda: owner._stop({child})) + except BaseException as error: # noqa: BLE001 — the probe must expose secondary native errors. + result = error + assert result is None, "reaping cannot replace an earlier operation failure" + assert calls == ["abort"] + finally: + child.cancel() + await asyncio.gather(child, return_exceptions=True) From 6a1d01819bad753473ff0c5420fec5bb877106d5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:34:22 +0000 Subject: [PATCH 071/181] Compose shared directory cleanup and foreground pool initialization Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 + server/router/directory.py | 116 ++++++--- .../test_directory_control_cleanup.py | 223 +++++++++++++++++- tests/unit/test_cell_control_census.py | 2 +- tests/unit/test_directory_control_cleanup.py | 191 ++++++++++++++- tests/unit/test_directory_pool_init.py | 160 +++++++++++++ 6 files changed, 655 insertions(+), 48 deletions(-) create mode 100644 tests/unit/test_directory_pool_init.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 1954a69930..c501cf0e4c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1279,6 +1279,17 @@ connection before releasing the pool context. Unexpected errors never wait for r on that discarded session. A late acquisition cannot start mutation work or obtain a fresh cleanup allowance; acquisition without a known proxy remains an unknown outcome. Ordinary live placement retains its separate transaction and connection handling. +Repeated interruption keeps the same close task and original end; a completed child +cancellation remains a terminal failure. Fallback abort errors cannot replace the first +operation or cleanup failure. + +The shared directory pool opens physical sessions only for foreground acquisitions; +pool object creation does not establish connectivity. Healthy released sessions are +cached under the existing size cap, recycling and idle expiry. A later cold acquisition +opens a new session. Fixed JSON codec initialization quarantines its never-borrowed raw +session on failure; native interruptions return to that foreground consumer after driver +cleanup, with no autonomous opening or replenishment task. If quarantine fails, the +original failure records that session closure is unknown. The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded diff --git a/server/router/directory.py b/server/router/directory.py index d4f3f62583..bdf9f4bdd6 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -152,13 +152,37 @@ def _require_dsn() -> str: return settings.router_pg_dsn +class _PoolInitFailure(Exception): + """Carry native control intent through the driver's ordinary-error cleanup.""" + + def __init__(self, error: SystemExit | KeyboardInterrupt) -> None: + super().__init__("directory codec initialization interrupted") + self.error = error + + async def _init(conn: Any) -> None: - for typ in ("json", "jsonb"): - await conn.set_type_codec(typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog") + try: + for typ in ("json", "jsonb"): + await conn.set_type_codec( + typ, encoder=json.dumps, decoder=json.loads, schema="pg_catalog" + ) + except BaseException as original: + # Only builtin codecs run here: this raw session has never been borrowed or + # held command locks. Quarantine before the driver can lose its ownership. + try: + conn.terminate() + except BaseException: # noqa: BLE001 — disposal must preserve the first init failure. + try: + original.add_note("directory initialization session closure is unknown") + except BaseException: # noqa: BLE001 — diagnostics must preserve the first failure. + pass + if isinstance(original, (SystemExit, KeyboardInterrupt)): + raise _PoolInitFailure(original) from None + raise async def pool() -> Any: - """The directory's pool, opened on first use.""" + """The directory's pool; physical sessions open only for foreground consumers.""" global _pool if _pool is not None: return _pool @@ -166,19 +190,22 @@ async def pool() -> Any: async with _opening: if _pool is None: - _pool = await asyncpg.create_pool( - _require_dsn(), - min_size=1, - max_size=POOL_MAX, - init=_init, - server_settings={ - **_SESSION, - "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), - "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), - }, - command_timeout=STATEMENT_TIMEOUT_S + 1, - timeout=STATEMENT_TIMEOUT_S, - ) + try: + _pool = await asyncpg.create_pool( + _require_dsn(), + min_size=0, + max_size=POOL_MAX, + init=_init, + server_settings={ + **_SESSION, + "statement_timeout": str(STATEMENT_TIMEOUT_S * 1000), + "idle_in_transaction_session_timeout": str(STATEMENT_TIMEOUT_S * 2000), + }, + command_timeout=STATEMENT_TIMEOUT_S + 1, + timeout=STATEMENT_TIMEOUT_S, + ) + except _PoolInitFailure as failed: + raise failed.error from None return _pool @@ -195,13 +222,17 @@ async def _connection(*, lookup: bool = False) -> AsyncIterator[Any]: slot, the pool's opening and the connection); a lookup's also takes one of LOOKUP_SLOTS. A wait past the bound raises TimeoutError, an OSError, which `_reading` answers as the directory unavailable.""" - async with AsyncExitStack() as stack: - async with asyncio.timeout(STATEMENT_TIMEOUT_S): - if lookup: - await _lookups.acquire() - stack.callback(_lookups.release) - conn = await stack.enter_async_context((await pool()).acquire()) - yield conn + try: + async with AsyncExitStack() as stack: + async with asyncio.timeout(STATEMENT_TIMEOUT_S): + if lookup: + await _lookups.acquire() + stack.callback(_lookups.release) + conn = await stack.enter_async_context((await pool()).acquire()) + yield conn + except _PoolInitFailure as failed: + # Driver acquisition and context cleanup finish before control intent returns. + raise failed.error from None @asynccontextmanager @@ -919,9 +950,9 @@ async def _control_result[T](work: Callable[[], Awaitable[T]]) -> T | _ControlFa class _ControlOwner: """Own only this operator attempt's proxy, through close and pool release.""" - def __init__(self) -> None: - self.end = time.monotonic() + CONTROL_MAX_S - self.work_end = self.end - min(1.0, CONTROL_MAX_S / 4) + def __init__(self, seconds: float) -> None: + self.end = time.monotonic() + seconds + self.work_end = self.end - min(1.0, seconds / 4) self.context = _connection() self.conn: Any = None self.entered = False @@ -949,12 +980,33 @@ async def close(self) -> None: return if self.close_task is None: self.close_task = self.start(lambda: pg.close_owned(self.conn, end=self.end)) + interrupted: asyncio.CancelledError | None = None try: - await self.wait(self.close_task, self.end) + while not self.close_task.done(): + remaining = self.end - time.monotonic() + if remaining <= 0: + raise TimeoutError("directory control deadline expired; outcome unknown") + try: + await asyncio.wait({self.close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Another signal cannot abandon the public cancellation in flight. + interrupted = interrupted or error + result = self.close_task.result() + if isinstance(result, _ControlFatal): + raise _ControlInterrupted(result) except BaseException: - pg._terminate(self.conn) - self.close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort must preserve the first cleanup failure. + pass + finally: + if not self.close_task.done(): + self.close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def release(self) -> None: if self.entered: @@ -1008,10 +1060,10 @@ def __init__(self, fatal: _ControlFatal) -> None: self.fatal = fatal -async def _control_transaction[T](change: Callable[[Any], Awaitable[T]]) -> T: +async def _control_transaction[T](change: Callable[[Any], Awaitable[T]], *, seconds: float) -> T: """Bound the whole callback, close first, then settle work and pool ownership.""" async with _reading(): - owner = _ControlOwner() + owner = _ControlOwner(seconds) task = owner.start(lambda: owner.work(change)) try: result = await owner.wait(task, owner.work_end) @@ -1050,7 +1102,7 @@ async def locked(conn: Any) -> T: await conn.execute("SELECT pg_logical_emit_message(true, 'loom.control', '')") return await change(conn, attaching) - return await _control_transaction(locked) + return await _control_transaction(locked, seconds=CONTROL_MAX_S) async def claim(cell_name: str, expected: Control) -> Attaching: diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index 9970d6b5ee..2df016414a 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -55,7 +55,9 @@ async def work(conn): seen.set() await conn.execute("SELECT pg_sleep(10)") - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) try: await asyncio.wait_for(seen.wait(), 2) await sleeping(directory_database, backend["pid"]) @@ -97,7 +99,7 @@ async def work(conn): raise original with pytest.raises(ValueError) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value is original and "rollback" not in backend assert time.monotonic() - began < 0.4 await released(directory_database, backend["pid"]) @@ -112,7 +114,10 @@ async def work(conn): await conn.execute("UPDATE probe SET id=2") return "acknowledged" - assert await directory._control_transaction(work) == "acknowledged" + assert ( + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + == "acknowledged" + ) assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -122,6 +127,7 @@ async def test_directory_second_cancel_keeps_first_and_close_precedes_worker_can ): import asyncpg + monkeypatch.setattr(directory, "CONTROL_MAX_S", 0.4) real_close = asyncpg.Connection.close seen, closing, allow_close = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} @@ -146,7 +152,9 @@ async def close(self, **kwargs): await real_close(self, **kwargs) monkeypatch.setattr(asyncpg.Connection, "close", close) - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) try: await asyncio.wait_for(seen.wait(), 2) await sleeping(directory_database, backend["pid"]) @@ -187,7 +195,7 @@ async def work(conn): monkeypatch.setattr(asyncpg.Connection, "execute", execute) with pytest.raises(directory.DirectoryUnavailable) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value.__cause__ is original assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -215,7 +223,7 @@ async def work(conn): monkeypatch.setattr(asyncpg.Connection, "close", close) with pytest.raises(ValueError) as caught: - await directory._control_transaction(work) + await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) assert caught.value is original assert await directory_database.fetchval("SELECT id FROM probe") == 2 await released(directory_database, backend["pid"]) @@ -267,7 +275,7 @@ async def work(conn): asyncpg.Connection.execute = execute directory._connection = connection try: - try: await directory._control_transaction(work) + try: await directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) except (SystemExit, KeyboardInterrupt) as caught: assert caught is original else: raise AssertionError('fatal guard returned success') if stage == 'acquire': @@ -294,3 +302,204 @@ async def work(conn): ) stdout, stderr = await asyncio.wait_for(process.communicate(), 10) assert process.returncode == 0, (stdout.decode(), stderr.decode()) + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("healthy_loan", [False, True]) +@pytest.mark.parametrize("driver_close", ["normal", "exit", "keyboard"]) +async def test_failed_codec_raw_session_is_disposed_without_touching_other_loans( + directory_database, monkeypatch, kind, healthy_loan, driver_close +): + import asyncpg + + original = { + "ordinary": ValueError("fixed codec failed"), + "cancel": asyncio.CancelledError("fixed codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("fixed codec interrupted"), + }[kind] + pool = await directory.pool() + assert pool.get_size() == 0 # Pool object creation does not open a backend. + healthy = await pool.acquire() if healthy_loan else None + healthy_pid = healthy.get_server_pid() if healthy else None + real_codec, real_close = asyncpg.Connection.set_type_codec, asyncpg.Connection.close + failed = {} + attempts = 0 + + async def codec(self, *args, **kwargs): + nonlocal attempts + attempts += 1 + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + async def close(self, **kwargs): + if self is failed.get("raw"): + # This is the failed, never-borrowed raw codec session, not an active + # command session: synchronous quarantine must precede driver cleanup. + assert self.is_closed() + failed["driver close"] = True + if driver_close == "exit": + raise SystemExit(21) + if driver_close == "keyboard": + raise KeyboardInterrupt("secondary closed-raw cleanup failure") + return await real_close(self, **kwargs) + + async def acquire(): + async with directory._connection() as conn: + pytest.fail(f"failed initializer admitted backend {conn.get_server_pid()}") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "close", close) + try: + if kind in {"exit", "keyboard"}: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + else: + with pytest.raises(type(original)) as caught: + await acquire() + assert caught.value is original + assert failed["driver close"] and attempts == 1 + await released(directory_database, failed["pid"]) + # Driver background replenishment must not swallow/retry control intent. + for _ in range(3): + await asyncio.sleep(0) + assert attempts == 1 + if healthy: + assert healthy.get_server_pid() == healthy_pid + assert await healthy.fetchval("SELECT 1") == 1 + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + async with directory._connection() as recovered: + recovered_pid = recovered.get_server_pid() + assert recovered_pid not in {failed["pid"], healthy_pid} + assert await recovered.fetchval("SELECT 1") == 1 + async with directory._connection() as cached: + assert cached.get_server_pid() == recovered_pid + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + if healthy: + await pool.release(healthy) + + +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "exit", "keyboard"]) +async def test_codec_quarantine_failure_keeps_native_original_and_unknown_diagnostic( + directory_database, monkeypatch, kind, secondary +): + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + second = { + "ordinary": ValueError("termination failed"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("termination interrupted"), + }[secondary] + real_codec, real_terminate = asyncpg.Connection.set_type_codec, asyncpg.Connection.terminate + failed = {} + + async def codec(self, *args, **kwargs): + failed["raw"], failed["pid"] = self, self.get_server_pid() + raise original + + def terminate(self): + if self is failed.get("raw"): + raise second + return real_terminate(self) + + async def acquire(): + async with directory._connection(): + pytest.fail("failed initialization returned success") + + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", terminate) + try: + result = await directory._control_result(acquire) + assert isinstance(result, directory._ControlFatal) and result.error is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + # Termination's request is not the proof; here the driver's subsequent + # ordinary-error public close really completes and the observer proves it. + await released(directory_database, failed["pid"]) + finally: + monkeypatch.setattr(asyncpg.Connection, "set_type_codec", real_codec) + monkeypatch.setattr(asyncpg.Connection, "terminate", real_terminate) + raw = failed.get("raw") + if raw is not None: + real_terminate(raw) + + +@pytest.mark.parametrize("signal_count", [1, 2]) +async def test_directory_signals_keep_real_cancel_request_alive_until_original_end( + directory_database, monkeypatch, signal_count +): + from asyncpg import connect_utils + + seen, cancelling, allow_cancel = asyncio.Event(), asyncio.Event(), asyncio.Event() + backend = {} + first_signal = {} + real_wait = directory._ControlOwner.wait + real_cancel = connect_utils._cancel + seconds = 0.5 + + async def cancel(**kwargs): + if kwargs["backend_pid"] == backend.get("pid"): + cancelling.set() + await allow_cancel.wait() + return await real_cancel(**kwargs) + + async def work(conn): + backend["pid"] = conn.get_server_pid() + await locked(conn) + seen.set() + await conn.execute("SELECT pg_sleep(10)") + + async def remember(self, task, end): + try: + return await real_wait(self, task, end) + except asyncio.CancelledError as error: + first_signal.setdefault("error", error) + raise + + monkeypatch.setattr(directory._ControlOwner, "wait", remember) + monkeypatch.setattr(connect_utils, "_cancel", cancel) + began = time.monotonic() + task = asyncio.create_task(directory._control_transaction(work, seconds=seconds)) + try: + await asyncio.wait_for(seen.wait(), 2) + await sleeping(directory_database, backend["pid"]) + task.cancel("first directory signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signal_count == 2: + task.cancel("second directory signal") + await asyncio.sleep(0.03) + assert not task.done(), "second signal abandoned a live server cancel request" + assert await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND wait_event='PgSleep')", + backend["pid"], + ) + assert ( + await directory_database.fetchval( + "SELECT count(*) FROM pg_locks WHERE pid=$1", backend["pid"] + ) + > 0 + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first directory signal") as caught: + await asyncio.wait_for(task, seconds) + assert caught.value is first_signal["error"] + assert time.monotonic() - began <= seconds + 0.1 + await released(directory_database, backend["pid"]) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if backend: + await directory_database.execute("SELECT pg_cancel_backend($1)", backend["pid"]) + async with asyncio.timeout(2): + for _ in range(200): + if not await directory_database.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", backend["pid"] + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("test-owned backend remained after exact cancellation") diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 45b8e85baa..64359d1293 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -411,7 +411,7 @@ def function(path: str, name: str) -> ast.AST: "server/router/directory.py", "_attach_change", [ - "_control_transaction(locked)", + "_control_transaction(locked, seconds=CONTROL_MAX_S)", "control_refusal(attaching, expected, claim=claim)", "change(conn, attaching)", ], diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py index 1a66b20c26..be6fee8329 100644 --- a/tests/unit/test_directory_control_cleanup.py +++ b/tests/unit/test_directory_control_cleanup.py @@ -29,7 +29,12 @@ async def test_success_closes_once_before_pool_exit(owned): events = [] conn.close.side_effect = lambda **_: events.append("close") context.__aexit__.side_effect = lambda *_: events.append("exit") - assert await directory._control_transaction(AsyncMock(return_value=19)) == 19 + assert ( + await directory._control_transaction( + AsyncMock(return_value=19), seconds=directory.CONTROL_MAX_S + ) + == 19 + ) assert events == ["close", "exit"] conn.close.assert_awaited_once() conn.transaction.return_value.commit.assert_awaited_once() @@ -41,7 +46,9 @@ async def test_original_error_survives_close_and_exit_failure(owned): conn.close.side_effect = RuntimeError("close failure") context.__aexit__.side_effect = LookupError("exit failure") with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock(side_effect=original)) + await directory._control_transaction( + AsyncMock(side_effect=original), seconds=directory.CONTROL_MAX_S + ) assert caught.value is original conn.close.assert_awaited_once() context.__aexit__.assert_awaited_once() @@ -54,7 +61,7 @@ async def test_close_failure_survives_exit_failure_after_acknowledged_commit(own conn.close.side_effect = original context.__aexit__.side_effect = RuntimeError("exit failure") with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert caught.value is original conn.transaction.return_value.commit.assert_awaited_once() @@ -70,7 +77,7 @@ async def stalled(*_): context.__aexit__.side_effect = stalled began = time.monotonic() with pytest.raises(directory.DirectoryUnavailable): - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert entered.is_set() and time.monotonic() - began < 0.2 conn.close.assert_awaited_once() @@ -80,7 +87,7 @@ async def test_acquisition_failure_does_not_close_an_unowned_proxy(owned): original = ValueError("acquisition failure") context.__aenter__.side_effect = original with pytest.raises(ValueError) as caught: - await directory._control_transaction(AsyncMock()) + await directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) assert caught.value is original conn.close.assert_not_awaited() conn.terminate.assert_not_called() @@ -96,7 +103,9 @@ async def acquire(): await asyncio.Future() context.__aenter__.side_effect = acquire - task = asyncio.create_task(directory._control_transaction(AsyncMock())) + task = asyncio.create_task( + directory._control_transaction(AsyncMock(), seconds=directory.CONTROL_MAX_S) + ) await entered.wait() task.cancel("first acquisition cancellation") with pytest.raises(asyncio.CancelledError, match="first acquisition cancellation"): @@ -105,7 +114,12 @@ async def acquire(): conn.terminate.assert_not_called() context.__aexit__.assert_not_awaited() context.__aenter__.side_effect = None - assert await directory._control_transaction(AsyncMock(return_value=2)) == 2 + assert ( + await directory._control_transaction( + AsyncMock(return_value=2), seconds=directory.CONTROL_MAX_S + ) + == 2 + ) conn.close.assert_awaited_once() @@ -122,7 +136,9 @@ async def acquire(): context.__aenter__.side_effect = acquire work = AsyncMock() - task = asyncio.create_task(directory._control_transaction(work)) + task = asyncio.create_task( + directory._control_transaction(work, seconds=directory.CONTROL_MAX_S) + ) await entered.wait() task.cancel("acquisition cancelled") with pytest.raises(asyncio.CancelledError, match="acquisition cancelled"): @@ -134,3 +150,162 @@ async def acquire(): conn.transaction.assert_not_called() conn.terminate.assert_called() conn.close.assert_not_awaited() # Expired original end cannot create a fresh close budget. + + +@pytest.mark.parametrize( + "ending", ["success", "failure", "self_cancel", "exit", "keyboard", "deadline"] +) +@pytest.mark.parametrize("locally_closed", [False, True]) +async def test_outer_signals_wait_for_the_same_close_and_keep_first( + owned, monkeypatch, ending, locally_closed +): + conn, _ = owned + owner = directory._ControlOwner(0.08 if ending == "deadline" else 10) + conn.is_closed.return_value = locally_closed + conn._lost = locally_closed + owner.conn = conn + end = owner.end + started, allow = asyncio.Event(), asyncio.Event() + first = asyncio.CancelledError("first outer signal") + failure = ValueError("close failure") + real_wait = asyncio.wait + waits = 0 + close_cancelled = [] + calls = [] + began = time.monotonic() + + async def close(_conn, *, end): + calls.append(_conn) + assert _conn is conn and end == owner.end + started.set() + try: + await allow.wait() + except asyncio.CancelledError: + close_cancelled.append(True) + raise + if ending == "failure": + raise failure + if ending == "self_cancel": + raise asyncio.CancelledError("close child cancelled itself") + if ending == "exit": + raise SystemExit(17) + if ending == "keyboard": + raise KeyboardInterrupt("close child interrupted") + + async def interrupted_wait(tasks, **kwargs): + nonlocal waits + waits += 1 + if waits == 1: + await started.wait() + raise first + return await real_wait(tasks, **kwargs) + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + task = asyncio.create_task(owner.close()) + await started.wait() + for _ in range(3): + await asyncio.sleep(0) + assert not task.done(), "an outer signal cancelled independent close before its end" + conn.terminate.assert_not_called() + assert not close_cancelled and owner.end == end + if ending != "deadline": + allow.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + assert caught.value is first and owner.end == end + assert time.monotonic() - began < 0.2 + assert calls == [conn] and waits < 5 + if ending == "success": + conn.terminate.assert_not_called() + else: + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("already_done", [False, True]) +async def test_self_cancelled_close_is_terminal_before_a_long_end(owned, monkeypatch, already_done): + conn, _ = owned + owner = directory._ControlOwner(10) + owner.conn = conn + original = asyncio.CancelledError("close child self-cancelled") + + async def close(_conn, *, end): + raise original + + monkeypatch.setattr(directory.pg, "close_owned", close) + if already_done: + owner.close_task = owner.start(lambda: close(conn, end=owner.end)) + await asyncio.sleep(0) + assert owner.close_task.done() + began = time.monotonic() + with pytest.raises(asyncio.CancelledError) as caught: + await owner.close() + assert caught.value is original + assert time.monotonic() - began < 0.2 + conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize( + "first_kind", ["ordinary", "cancel", "exit", "keyboard", "outer_cancel", "deadline"] +) +async def test_secondary_abort_never_replaces_first_or_skips_owned_task_settlement( + owned, monkeypatch, abort_kind, first_kind +): + conn, _ = owned + owner = directory._ControlOwner(0.02 if first_kind == "deadline" else 10) + owner.conn = conn + errors = { + "ordinary": ValueError("first close failure"), + "cancel": asyncio.CancelledError("first close cancellation"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("first close interruption"), + "outer_cancel": asyncio.CancelledError("first outer cancellation"), + } + first = errors.get(first_kind) + secondary = { + "ordinary": RuntimeError("secondary abort failure"), + "cancel": asyncio.CancelledError("secondary abort cancellation"), + "exit": SystemExit(19), + "keyboard": KeyboardInterrupt("secondary abort interruption"), + }[abort_kind] + conn.terminate.side_effect = secondary + began = asyncio.Event() + cancelled = asyncio.Event() + real_wait = asyncio.wait + signals = 0 + + async def close(_conn, *, end): + began.set() + if first_kind == "deadline": + try: + await asyncio.Future() + finally: + cancelled.set() + raise ValueError("later close failure") if first_kind == "outer_cancel" else first + + async def interrupted_wait(tasks, **kwargs): + nonlocal signals + signals += 1 + if first_kind == "outer_cancel" and signals == 1: + raise first + return await real_wait(tasks, **kwargs) + + async def observed(): + try: + await owner.close() + except BaseException as error: # noqa: BLE001 — test native failures must remain inert too. + return error.fatal.error if isinstance(error, directory._ControlInterrupted) else error + pytest.fail("failed required cleanup became success") + + monkeypatch.setattr(directory.pg, "close_owned", close) + monkeypatch.setattr(asyncio, "wait", interrupted_wait) + result = await observed() + if first_kind == "deadline": + assert isinstance(result, TimeoutError) + await asyncio.wait_for(cancelled.wait(), 0.1) + assert owner.close_task.cancelled() + else: + assert result is first + assert began.is_set() + conn.terminate.assert_called_once_with() diff --git a/tests/unit/test_directory_pool_init.py b/tests/unit/test_directory_pool_init.py new file mode 100644 index 0000000000..ea8bb3fd37 --- /dev/null +++ b/tests/unit/test_directory_pool_init.py @@ -0,0 +1,160 @@ +"""Failed fixed-codec initialization never admits its unborrowed raw connection.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.router import directory + + +@pytest.mark.parametrize( + "error", [ValueError("codec failure"), asyncio.CancelledError("codec cancelled")] +) +@pytest.mark.parametrize("position", [0, 1]) +async def test_every_failed_initialization_quarantines_its_exact_raw_session(error, position): + conn = Mock(set_type_codec=AsyncMock(side_effect=[None] * position + [error])) + with pytest.raises(type(error)) as caught: + await directory._init(conn) + assert caught.value is error + conn.terminate.assert_called_once_with() + assert conn.set_type_codec.await_count == position + 1 + + +async def test_fixed_builtin_codecs_do_not_execute_sql(): + import json + from types import MethodType, SimpleNamespace + + import asyncpg + + conn = SimpleNamespace( + _check_open=Mock(), + _protocol=Mock(), + _drop_local_statement_cache=Mock(), + _execute=AsyncMock(side_effect=AssertionError("codec initialization executed SQL")), + terminate=Mock(), + ) + conn._introspect_type = MethodType(asyncpg.Connection._introspect_type, conn) + conn.set_type_codec = MethodType(asyncpg.Connection.set_type_codec, conn) + await directory._init(conn) + conn._execute.assert_not_awaited() + conn.terminate.assert_not_called() + calls = conn._protocol.get_settings.return_value.add_python_codec.call_args_list + assert len(calls) == 2 + assert [call.args[1] for call in calls] == ["json", "jsonb"] + assert all(call.args[2] == "pg_catalog" for call in calls) + assert all(call.args[5:7] == (json.dumps, json.loads) for call in calls) + + +@pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("secondary", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_secondary_quarantine_failure_keeps_first_and_reports_unknown(kind, secondary): + def error(which): + return { + "ordinary": ValueError("original codec failure"), + "cancel": asyncio.CancelledError("codec cancelled"), + "exit": SystemExit(17), + "keyboard": KeyboardInterrupt("codec interrupted"), + }[which] + + original = error(kind) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=error(secondary)) + ) + if isinstance(original, (SystemExit, KeyboardInterrupt)): + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + else: + with pytest.raises(type(original)) as caught: + await directory._init(conn) + assert caught.value is original + assert original.__notes__ == ["directory initialization session closure is unknown"] + conn.terminate.assert_called_once_with() + conn.set_type_codec.assert_awaited_once() + + +async def test_diagnostic_failure_cannot_replace_original_native_failure(): + class BrokenNote(SystemExit): + def add_note(self, note): + raise KeyboardInterrupt("secondary diagnostic failure") + + original = BrokenNote(17) + conn = Mock( + set_type_codec=AsyncMock(side_effect=original), terminate=Mock(side_effect=RuntimeError) + ) + with pytest.raises(directory._PoolInitFailure) as caught: + await directory._init(conn) + assert caught.value.error is original + + +async def test_pool_has_no_autonomous_opening_and_keeps_existing_limits(monkeypatch): + import asyncpg + + opened = object() + create = AsyncMock(return_value=opened) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + monkeypatch.setattr(asyncpg, "create_pool", create) + assert await directory.pool() is opened + assert await directory.pool() is opened + create.assert_awaited_once_with( + "private-test-dsn", + min_size=0, + max_size=10, + init=directory._init, + server_settings={ + **directory._SESSION, + "statement_timeout": "10000", + "idle_in_transaction_session_timeout": "20000", + }, + command_timeout=11, + timeout=10, + ) + + +@pytest.mark.parametrize("boundary", ["creation", "acquisition", "lookup"]) +@pytest.mark.parametrize("kind", ["exit", "keyboard"]) +async def test_every_pool_boundary_unwraps_identical_native_after_driver_cleanup( + monkeypatch, boundary, kind +): + from contextlib import asynccontextmanager + + import asyncpg + + original = SystemExit(17) if kind == "exit" else KeyboardInterrupt("codec interrupted") + events = [] + carrier = directory._PoolInitFailure(original) + monkeypatch.setattr(directory, "_pool", None) + monkeypatch.setattr(directory, "_require_dsn", lambda: "private-test-dsn") + + async def create(*args, **kwargs): + events.append("driver creation cleanup") + raise carrier + + @asynccontextmanager + async def acquire(): + try: + raise carrier + yield # pragma: no cover + finally: + events.append("driver acquisition cleanup") + + if boundary == "creation": + monkeypatch.setattr(asyncpg, "create_pool", create) + else: + monkeypatch.setattr(directory, "_pool", Mock(acquire=acquire)) + slots = directory._lookups._value + with pytest.raises(type(original)) as caught: + if boundary == "creation": + await directory.pool() + else: + async with directory._connection(lookup=boundary == "lookup"): + pytest.fail("failed initializer admitted a connection") + assert caught.value is original + assert events == [ + "driver creation cleanup" if boundary == "creation" else "driver acquisition cleanup" + ] + assert directory._lookups._value == slots From cdce8ec301c3e2446781be6add0145469de3aaa2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:21:49 +0000 Subject: [PATCH 072/181] Keep owned cancellation alive through repeated signals Signed-off-by: zlareb1 (cherry picked from commit 96df6ab6bc91198f867d0c2acedcb0aca0e044c8) --- server/core/pg.py | 26 +++++- tests/integration/test_pg_owned_cleanup.py | 58 ++++++++++++ tests/unit/test_pg_close_owned.py | 101 ++++++++++++++++++++- 3 files changed, 179 insertions(+), 6 deletions(-) diff --git a/server/core/pg.py b/server/core/pg.py index 1e51975dfb..8f1d8192c7 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -402,23 +402,41 @@ async def close() -> SystemExit | KeyboardInterrupt | None: return None task: asyncio.Task[Any] | None = None + interrupted: asyncio.CancelledError | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") task = asyncio.create_task(close()) - done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - if not done: - raise TimeoutError("Postgres owned-session cleanup deadline expired") + while True: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + try: + done, _ = await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as error: + # Keep cancellation delivery alive; another signal cannot reset its end. + interrupted = interrupted or error + continue + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + break if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): raise result except BaseException: - _terminate(conn) + try: + _terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first cleanup failure. + pass if task is not None: if not task.done(): task.cancel() task.add_done_callback(_retrieve_task) + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _release(pool: Any, conn: Any) -> None: diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py index 9085fce011..62a339774a 100644 --- a/tests/integration/test_pg_owned_cleanup.py +++ b/tests/integration/test_pg_owned_cleanup.py @@ -75,6 +75,64 @@ async def connect() -> AsyncIterator[Any]: await released(observer, pid) +@pytest.mark.parametrize("signals", [1, 2]) +async def test_owned_migrator_keeps_real_cancel_request_alive( + database, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, signals: int +) -> None: + from asyncpg import connect_utils + + conn, observer = database + pid = conn.get_server_pid() + cancelling, allow_cancel = asyncio.Event(), asyncio.Event() + real_cancel = connect_utils._cancel + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.5) + + async def held_cancel(**kwargs: Any) -> None: + if kwargs["backend_pid"] == pid: + cancelling.set() + await allow_cancel.wait() + await real_cancel(**kwargs) + + monkeypatch.setattr(connect_utils, "_cancel", held_cancel) + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + try: + await sleeping(observer, pid) + task.cancel("first signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signals == 2: + task.cancel("second signal") + await asyncio.sleep(0.03) + assert not task.done(), ( + await observer.fetchval("SELECT wait_event FROM pg_stat_activity WHERE pid=$1", pid), + await observer.fetchval("SELECT count(*) FROM pg_locks WHERE pid=$1", pid), + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await asyncio.wait_for(task, 0.5) + await released(observer, pid) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await observer.execute("SELECT pg_cancel_backend($1)", pid) + async with asyncio.timeout(2): + for _ in range(200): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("owned backend did not end after teardown cancellation") + + @pytest.mark.parametrize("transaction_context", [False, True]) async def test_public_owned_close_releases_live_query_before_task_cancel( database, transaction_context: bool diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index 1ba7ab1f6a..eba8142aff 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -53,7 +53,102 @@ async def test_close_timeout_aborts_within_the_original_deadline() -> None: assert conn.aborted and time.monotonic() - started < 0.3 -async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: +async def test_outer_signals_do_not_abort_pending_public_close() -> None: + conn = Connection() + task = asyncio.create_task(pg.close_owned(conn, end=time.monotonic() + 0.5)) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + task.cancel("second close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + conn.finished.set() + with pytest.raises(asyncio.CancelledError, match="first close interruption"): + await task + assert not conn.aborted + assert len(conn.timeouts) == 1 + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_self_cancelled_close_child_does_not_spin_to_deadline() -> None: + class SelfCancelled(Connection): + async def close(self, **_: Any) -> None: + raise asyncio.CancelledError("close child cancelled") + + started = time.monotonic() + with pytest.raises(asyncio.CancelledError, match="close child cancelled"): + await pg.close_owned(SelfCancelled(), end=started + 10) + assert time.monotonic() - started < 0.3 + + +@pytest.mark.parametrize( + "later", + [ + ValueError("late close"), + asyncio.CancelledError("close child"), + SystemExit(17), + KeyboardInterrupt(), + ], +) +async def test_first_outer_cancel_survives_a_later_close_failure(later: BaseException) -> None: + class Failing(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + await self.finished.wait() + raise later + + def is_closed(self) -> bool: + return True # An aborted transport cannot prove cancellation acknowledgement. + + conn = Failing() + + async def captured() -> BaseException | None: + try: + await pg.close_owned(conn, end=time.monotonic() + 10) + except BaseException as error: # noqa: BLE001 — keep native failures inside the test task. + return error + return None + + task = asyncio.create_task(captured()) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + started = time.monotonic() + conn.finished.set() + result = await task + assert isinstance(result, asyncio.CancelledError) + assert str(result) == "first close interruption" + assert time.monotonic() - started < 0.3 + assert conn.aborted + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_error", [ValueError("abort"), SystemExit(17), KeyboardInterrupt()]) +async def test_abort_failure_cannot_replace_the_close_failure(abort_error: BaseException) -> None: + original = ValueError("original close failure") + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise original + + def terminate(self) -> None: + raise abort_error + + with pytest.raises(ValueError) as result: + await pg.close_owned(Broken(), end=time.monotonic() + 1) + assert result.value is original + + +async def test_second_cancellation_waits_for_the_original_cleanup_end() -> None: conn = Connection() supervisor_started = asyncio.Event() original = asyncio.CancelledError("operation cancelled") @@ -64,7 +159,7 @@ async def supervisor() -> None: await asyncio.Future() except asyncio.CancelledError: try: - await pg.close_owned(conn, end=time.monotonic() + 1) + await pg.close_owned(conn, end=time.monotonic() + 0.03) except BaseException: # noqa: BLE001 — preserve the original cancellation. pass raise original from None @@ -74,6 +169,8 @@ async def supervisor() -> None: task.cancel() await asyncio.wait_for(conn.started.wait(), 1) task.cancel() + await asyncio.sleep(0) + assert not conn.aborted with pytest.raises(asyncio.CancelledError, match="operation cancelled"): await asyncio.wait_for(task, 0.3) assert conn.aborted From 70c21f2cd4312a5d6c8d210ec89d73114d9fccf4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:31:46 +0000 Subject: [PATCH 073/181] Migration fallback aborts preserve the first failure Signed-off-by: zlareb1 (cherry picked from commit 30a4ac5d4a0f01045db153af727ca11e4ed9066f) --- server/core/schema/pg_migrate.py | 10 +++- tests/unit/test_pg_close_owned.py | 89 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 1722da5a9b..a62b467c0c 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -152,7 +152,10 @@ async def _failed_migration( try: await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass release = asyncio.create_task( _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) @@ -162,7 +165,10 @@ async def _failed_migration( raise TimeoutError("migration context cleanup deadline expired") release.result() except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass finally: if not release.done(): release.cancel() diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index eba8142aff..cb5abfa58f 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -314,3 +314,92 @@ async def work(*_: Any) -> list[str]: ) assert len(conn.timeouts) == 1 assert conn.aborted + + +@pytest.mark.parametrize("seam", ["reap", "exit"]) +@pytest.mark.parametrize("original_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_migration_abort_preserves_first_and_finishes_context( + monkeypatch, tmp_path, seam, original_kind, abort_kind +): + from server.core.schema import pg_migrate + + def error(kind, message): + return { + "ordinary": ValueError(message), + "cancel": asyncio.CancelledError(message), + "exit": SystemExit(message), + "keyboard": KeyboardInterrupt(message), + }[kind] + + original, secondary = ( + error(original_kind, "migration failed"), + error(abort_kind, "abort failed"), + ) + events, first, exits, ends, retrieved, triples = [], [], [], [], [], [] + conn = Connection() + real_wait, real_failed, real_retrieve = ( + asyncio.wait, + pg_migrate._failed_migration, + pg._retrieve_task, + ) + + class Context: + async def __aenter__(self): + return conn + + async def __aexit__(self, kind, value, traceback): + exits.append((kind, value, traceback)) + events.append("exit") + if seam == "exit": + raise OSError("context exit failed") + + async def migrate(*args): + raise original + + async def close(_conn, *, end): + assert _conn is conn + ends.append(end) + events.append("close") + + async def failed(cm, actual_conn, task, observed, end): + first.append(observed) + triples.append((type(observed), observed, observed.__traceback__)) + return await real_failed(cm, actual_conn, task, observed, end) + + async def wait(tasks, *, timeout): # noqa: ASYNC109 — asyncio.wait test seam. + assert 0 <= timeout <= max(0, ends[0] - time.monotonic()) + 0.01 + if seam == "reap" and "reap" not in events: + events.append("reap") + raise asyncio.CancelledError("second signal during reap") + return await real_wait(tasks, timeout=timeout) + + def abort(): + events.append("abort") + raise secondary + + def retrieve(task): + retrieved.append(task) + real_retrieve(task) + + monkeypatch.setattr(pg_migrate, "_migrate", migrate) + monkeypatch.setattr(pg_migrate, "_failed_migration", failed) + monkeypatch.setattr(pg, "close_owned", close) + monkeypatch.setattr(conn, "terminate", abort) + monkeypatch.setattr(asyncio, "wait", wait) + monkeypatch.setattr(pg, "_retrieve_task", retrieve) + try: + await pg_migrate.migrate(tmp_path, connect=Context, schema="test") + except BaseException as observed: # noqa: BLE001 — native test failures must remain inert. + result = observed + else: + pytest.fail("failed migration returned success") + assert len(first) == 1 and result is first[0] + if original_kind != "cancel": + assert result is original + assert exits == triples + assert len(ends) == 1 and events.count("close") == events.count("abort") == 1 + assert events[-1] == ("exit" if seam == "reap" else "abort") + for _ in range(3): + await asyncio.sleep(0) + assert len(retrieved) == 2 and all(task.done() for task in retrieved) From 9d18614e60c404a998e8bb656a8d6f947259e96f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:21:49 +0000 Subject: [PATCH 074/181] Keep owned cancellation alive through repeated signals Signed-off-by: zlareb1 (cherry picked from commit 96df6ab6bc91198f867d0c2acedcb0aca0e044c8) --- server/core/pg.py | 26 +++++- tests/integration/test_pg_owned_cleanup.py | 58 ++++++++++++ tests/unit/test_pg_close_owned.py | 101 ++++++++++++++++++++- 3 files changed, 179 insertions(+), 6 deletions(-) diff --git a/server/core/pg.py b/server/core/pg.py index e70c702582..f672c9edde 100644 --- a/server/core/pg.py +++ b/server/core/pg.py @@ -402,23 +402,41 @@ async def close() -> SystemExit | KeyboardInterrupt | None: return None task: asyncio.Task[Any] | None = None + interrupted: asyncio.CancelledError | None = None try: remaining = end - time.monotonic() if remaining <= 0: raise TimeoutError("Postgres owned-session cleanup deadline expired") task = asyncio.create_task(close()) - done, _ = await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) - if not done: - raise TimeoutError("Postgres owned-session cleanup deadline expired") + while True: + remaining = end - time.monotonic() + if remaining <= 0: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + try: + done, _ = await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as error: + # Keep cancellation delivery alive; another signal cannot reset its end. + interrupted = interrupted or error + continue + if not done: + raise TimeoutError("Postgres owned-session cleanup deadline expired") + break if isinstance(result := task.result(), (SystemExit, KeyboardInterrupt)): raise result except BaseException: - _terminate(conn) + try: + _terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first cleanup failure. + pass if task is not None: if not task.done(): task.cancel() task.add_done_callback(_retrieve_task) + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _release(pool: Any, conn: Any) -> None: diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py index 9085fce011..62a339774a 100644 --- a/tests/integration/test_pg_owned_cleanup.py +++ b/tests/integration/test_pg_owned_cleanup.py @@ -75,6 +75,64 @@ async def connect() -> AsyncIterator[Any]: await released(observer, pid) +@pytest.mark.parametrize("signals", [1, 2]) +async def test_owned_migrator_keeps_real_cancel_request_alive( + database, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, signals: int +) -> None: + from asyncpg import connect_utils + + conn, observer = database + pid = conn.get_server_pid() + cancelling, allow_cancel = asyncio.Event(), asyncio.Event() + real_cancel = connect_utils._cancel + monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.5) + + async def held_cancel(**kwargs: Any) -> None: + if kwargs["backend_pid"] == pid: + cancelling.set() + await allow_cancel.wait() + await real_cancel(**kwargs) + + monkeypatch.setattr(connect_utils, "_cancel", held_cancel) + (tmp_path / "001_sleep.sql").write_text("SELECT * FROM probe FOR UPDATE; SELECT pg_sleep(10);") + + @contextlib.asynccontextmanager + async def connect() -> AsyncIterator[Any]: + yield conn + + task = asyncio.create_task(pg_migrate.migrate(tmp_path, connect=connect, schema="test")) + try: + await sleeping(observer, pid) + task.cancel("first signal") + await asyncio.wait_for(cancelling.wait(), 2) + if signals == 2: + task.cancel("second signal") + await asyncio.sleep(0.03) + assert not task.done(), ( + await observer.fetchval("SELECT wait_event FROM pg_stat_activity WHERE pid=$1", pid), + await observer.fetchval("SELECT count(*) FROM pg_locks WHERE pid=$1", pid), + ) + allow_cancel.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await asyncio.wait_for(task, 0.5) + await released(observer, pid) + finally: + allow_cancel.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await observer.execute("SELECT pg_cancel_backend($1)", pid) + async with asyncio.timeout(2): + for _ in range(200): + if not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", pid + ): + break + await asyncio.sleep(0.01) + else: + pytest.fail("owned backend did not end after teardown cancellation") + + @pytest.mark.parametrize("transaction_context", [False, True]) async def test_public_owned_close_releases_live_query_before_task_cancel( database, transaction_context: bool diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index 1ba7ab1f6a..eba8142aff 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -53,7 +53,102 @@ async def test_close_timeout_aborts_within_the_original_deadline() -> None: assert conn.aborted and time.monotonic() - started < 0.3 -async def test_second_cancellation_aborts_without_waiting_for_close_unwind() -> None: +async def test_outer_signals_do_not_abort_pending_public_close() -> None: + conn = Connection() + task = asyncio.create_task(pg.close_owned(conn, end=time.monotonic() + 0.5)) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + task.cancel("second close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + conn.finished.set() + with pytest.raises(asyncio.CancelledError, match="first close interruption"): + await task + assert not conn.aborted + assert len(conn.timeouts) == 1 + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_self_cancelled_close_child_does_not_spin_to_deadline() -> None: + class SelfCancelled(Connection): + async def close(self, **_: Any) -> None: + raise asyncio.CancelledError("close child cancelled") + + started = time.monotonic() + with pytest.raises(asyncio.CancelledError, match="close child cancelled"): + await pg.close_owned(SelfCancelled(), end=started + 10) + assert time.monotonic() - started < 0.3 + + +@pytest.mark.parametrize( + "later", + [ + ValueError("late close"), + asyncio.CancelledError("close child"), + SystemExit(17), + KeyboardInterrupt(), + ], +) +async def test_first_outer_cancel_survives_a_later_close_failure(later: BaseException) -> None: + class Failing(Connection): + async def close(self, **_: Any) -> None: + self.started.set() + await self.finished.wait() + raise later + + def is_closed(self) -> bool: + return True # An aborted transport cannot prove cancellation acknowledgement. + + conn = Failing() + + async def captured() -> BaseException | None: + try: + await pg.close_owned(conn, end=time.monotonic() + 10) + except BaseException as error: # noqa: BLE001 — keep native failures inside the test task. + return error + return None + + task = asyncio.create_task(captured()) + try: + await conn.started.wait() + task.cancel("first close interruption") + await asyncio.sleep(0) + assert not task.done() + assert not conn.aborted + started = time.monotonic() + conn.finished.set() + result = await task + assert isinstance(result, asyncio.CancelledError) + assert str(result) == "first close interruption" + assert time.monotonic() - started < 0.3 + assert conn.aborted + finally: + conn.finished.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_error", [ValueError("abort"), SystemExit(17), KeyboardInterrupt()]) +async def test_abort_failure_cannot_replace_the_close_failure(abort_error: BaseException) -> None: + original = ValueError("original close failure") + + class Broken(Connection): + async def close(self, **_: Any) -> None: + raise original + + def terminate(self) -> None: + raise abort_error + + with pytest.raises(ValueError) as result: + await pg.close_owned(Broken(), end=time.monotonic() + 1) + assert result.value is original + + +async def test_second_cancellation_waits_for_the_original_cleanup_end() -> None: conn = Connection() supervisor_started = asyncio.Event() original = asyncio.CancelledError("operation cancelled") @@ -64,7 +159,7 @@ async def supervisor() -> None: await asyncio.Future() except asyncio.CancelledError: try: - await pg.close_owned(conn, end=time.monotonic() + 1) + await pg.close_owned(conn, end=time.monotonic() + 0.03) except BaseException: # noqa: BLE001 — preserve the original cancellation. pass raise original from None @@ -74,6 +169,8 @@ async def supervisor() -> None: task.cancel() await asyncio.wait_for(conn.started.wait(), 1) task.cancel() + await asyncio.sleep(0) + assert not conn.aborted with pytest.raises(asyncio.CancelledError, match="operation cancelled"): await asyncio.wait_for(task, 0.3) assert conn.aborted From 0835f33b6b759fa22e8609125b6f016338cb18b6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:31:46 +0000 Subject: [PATCH 075/181] Migration fallback aborts preserve the first failure Signed-off-by: zlareb1 (cherry picked from commit 30a4ac5d4a0f01045db153af727ca11e4ed9066f) --- server/core/schema/pg_migrate.py | 10 +++- tests/unit/test_pg_close_owned.py | 89 +++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index 1722da5a9b..a62b467c0c 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -152,7 +152,10 @@ async def _failed_migration( try: await asyncio.wait({task}, timeout=max(0, end - time.monotonic())) except BaseException: # noqa: BLE001 — preserve the first failure on repeated cancellation. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass release = asyncio.create_task( _owned_result(lambda: cm.__aexit__(type(error), error, error.__traceback__)) ) @@ -162,7 +165,10 @@ async def _failed_migration( raise TimeoutError("migration context cleanup deadline expired") release.result() except BaseException: # noqa: BLE001 — a context exit cannot replace the migration failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — fallback abort cannot replace the migration failure. + pass finally: if not release.done(): release.cancel() diff --git a/tests/unit/test_pg_close_owned.py b/tests/unit/test_pg_close_owned.py index eba8142aff..cb5abfa58f 100644 --- a/tests/unit/test_pg_close_owned.py +++ b/tests/unit/test_pg_close_owned.py @@ -314,3 +314,92 @@ async def work(*_: Any) -> list[str]: ) assert len(conn.timeouts) == 1 assert conn.aborted + + +@pytest.mark.parametrize("seam", ["reap", "exit"]) +@pytest.mark.parametrize("original_kind", ["ordinary", "cancel", "exit", "keyboard"]) +@pytest.mark.parametrize("abort_kind", ["ordinary", "cancel", "exit", "keyboard"]) +async def test_failed_migration_abort_preserves_first_and_finishes_context( + monkeypatch, tmp_path, seam, original_kind, abort_kind +): + from server.core.schema import pg_migrate + + def error(kind, message): + return { + "ordinary": ValueError(message), + "cancel": asyncio.CancelledError(message), + "exit": SystemExit(message), + "keyboard": KeyboardInterrupt(message), + }[kind] + + original, secondary = ( + error(original_kind, "migration failed"), + error(abort_kind, "abort failed"), + ) + events, first, exits, ends, retrieved, triples = [], [], [], [], [], [] + conn = Connection() + real_wait, real_failed, real_retrieve = ( + asyncio.wait, + pg_migrate._failed_migration, + pg._retrieve_task, + ) + + class Context: + async def __aenter__(self): + return conn + + async def __aexit__(self, kind, value, traceback): + exits.append((kind, value, traceback)) + events.append("exit") + if seam == "exit": + raise OSError("context exit failed") + + async def migrate(*args): + raise original + + async def close(_conn, *, end): + assert _conn is conn + ends.append(end) + events.append("close") + + async def failed(cm, actual_conn, task, observed, end): + first.append(observed) + triples.append((type(observed), observed, observed.__traceback__)) + return await real_failed(cm, actual_conn, task, observed, end) + + async def wait(tasks, *, timeout): # noqa: ASYNC109 — asyncio.wait test seam. + assert 0 <= timeout <= max(0, ends[0] - time.monotonic()) + 0.01 + if seam == "reap" and "reap" not in events: + events.append("reap") + raise asyncio.CancelledError("second signal during reap") + return await real_wait(tasks, timeout=timeout) + + def abort(): + events.append("abort") + raise secondary + + def retrieve(task): + retrieved.append(task) + real_retrieve(task) + + monkeypatch.setattr(pg_migrate, "_migrate", migrate) + monkeypatch.setattr(pg_migrate, "_failed_migration", failed) + monkeypatch.setattr(pg, "close_owned", close) + monkeypatch.setattr(conn, "terminate", abort) + monkeypatch.setattr(asyncio, "wait", wait) + monkeypatch.setattr(pg, "_retrieve_task", retrieve) + try: + await pg_migrate.migrate(tmp_path, connect=Context, schema="test") + except BaseException as observed: # noqa: BLE001 — native test failures must remain inert. + result = observed + else: + pytest.fail("failed migration returned success") + assert len(first) == 1 and result is first[0] + if original_kind != "cancel": + assert result is original + assert exits == triples + assert len(ends) == 1 and events.count("close") == events.count("abort") == 1 + assert events[-1] == ("exit" if seam == "reap" else "abort") + for _ in range(3): + await asyncio.sleep(0) + assert len(retrieved) == 2 and all(task.done() for task in retrieved) From 723109d27c75236e2d719fd998ac7d3dbf6d165a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:40:26 +0000 Subject: [PATCH 076/181] Compose native control ownership with bounded abort cleanup Signed-off-by: zlareb1 --- server/cli/cell_control.py | 95 ++++--- .../integration/test_cell_control_cleanup.py | 12 +- tests/unit/test_cell_attach_cleanup.py | 10 +- tests/unit/test_cell_control_abort.py | 185 ++++++++++++++ tests/unit/test_cell_place_owner.py | 233 ++++++++++++++++++ 5 files changed, 497 insertions(+), 38 deletions(-) create mode 100644 tests/unit/test_cell_control_abort.py create mode 100644 tests/unit/test_cell_place_owner.py diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index c4360839ed..1966b9e62b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -119,7 +119,8 @@ async def _captured[T](work: Callable[[], Awaitable[T]]) -> T | _Fatal: class Owner: """One raw backend and one cleanup owner for a native control command.""" - def __init__(self) -> None: + def __init__(self, *, exit_s: float = pg._RELEASE_TIMEOUT_S) -> None: + self.exit_s = exit_s self.conn: Any = None self.phase_end: float | None = None self.close_end: float | None = None @@ -155,7 +156,7 @@ async def _open(self, what: str) -> None: async def close(self) -> None: if not self.closed: self.closed = True - self.close_end = self.phase_end or (time.monotonic() + pg._RELEASE_TIMEOUT_S) + self.close_end = self.phase_end or (time.monotonic() + self.exit_s) if self.conn is None: return if self._close_task is None: @@ -163,20 +164,33 @@ async def close(self) -> None: _captured(lambda: pg.close_owned(self.conn, end=cast(float, self.close_end))) ) self._close_task.add_done_callback(pg._retrieve_task) + interrupted: asyncio.CancelledError | None = None try: - done, _ = await asyncio.wait( - {self._close_task}, timeout=max(0, cast(float, self.close_end) - time.monotonic()) - ) - if not done: - raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + while not self._close_task.done(): + remaining = cast(float, self.close_end) - time.monotonic() + if remaining <= 0: + raise TimeoutError("cell control cleanup deadline expired; outcome unknown") + try: + await asyncio.wait({self._close_task}, timeout=remaining) + except asyncio.CancelledError as error: + # Signals cannot abort the public cancellation already in flight. + interrupted = interrupted or error result = self._close_task.result() if isinstance(result, _Fatal): raise _ForwardFatal(result) except BaseException: - pg._terminate(self.conn) - if not self._close_task.done(): - self._close_task.cancel() + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — abort cannot replace the first close failure. + pass + finally: + if not self._close_task.done(): + self._close_task.cancel() + if interrupted is not None: + raise interrupted from None raise + if interrupted is not None: + raise interrupted async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: for task in tasks: @@ -187,7 +201,10 @@ async def _stop(self, tasks: set[asyncio.Task[Any]]) -> None: await asyncio.wait(tasks, timeout=max(0, (self.close_end or 0) - time.monotonic())) except BaseException: # noqa: BLE001 — cleanup cannot replace the original failure. if self.conn is not None: - pg._terminate(self.conn) + try: + pg._terminate(self.conn) + except BaseException: # noqa: BLE001 — failed abort preserves the caller's failure. + pass async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> T: if self.closed or self.phase_end is not None: @@ -219,31 +236,43 @@ async def call[T](self, work: Callable[[], Awaitable[T]], *, total_s: float) -> async def transaction[T](self, work: Callable[[], Awaitable[T]]) -> T: return await self.call(lambda: transaction(self.conn, work), total_s=TRANSACTION_S) + async def run[T]( + self, + opening: Callable[[], Awaitable[None]], + work: Callable[[], Awaitable[T]], + *, + opening_s: float, + ) -> T: + """Own one command; fatal guard failures leave only after bounded cleanup.""" + if self.conn is not None or self.phase_end is not None or self._tasks or self.closed: + raise _refused("a command needs a fresh control owner") + try: + await self.call(opening, total_s=opening_s) + result = await asyncio.shield(self._start(work)) + if isinstance(result, _Fatal): + raise _ForwardFatal(result) + except BaseException as original: + try: + await self.close() + except BaseException: # noqa: BLE001 — preserve the command's original failure. + pass + await self._stop(set(self._tasks)) + if isinstance(original, _ForwardFatal): + raise original.fatal.error from None + raise + try: + await self.close() + except _ForwardFatal as interrupted: + raise interrupted.fatal.error from None + finally: + await self._stop(set(self._tasks)) + return result + async def run[T](what: str, work: Callable[[Owner], Awaitable[T]]) -> T: - """Run at the command boundary; fatal guard exceptions leave only after cleanup.""" + """Run native control with its existing opening, transaction and exit allowances.""" owner = Owner() - try: - await owner.call(lambda: owner._open(what), total_s=OPEN_S) - result = await asyncio.shield(owner._start(lambda: work(owner))) - if isinstance(result, _Fatal): - raise _ForwardFatal(result) - except BaseException as original: - try: - await owner.close() - except BaseException: # noqa: BLE001 — preserve the command's original failure. - pass - await owner._stop(set(owner._tasks)) - if isinstance(original, _ForwardFatal): - raise original.fatal.error from None - raise - try: - await owner.close() - except _ForwardFatal as interrupted: - raise interrupted.fatal.error from None - finally: - await owner._stop(set(owner._tasks)) - return result + return await owner.run(lambda: owner._open(what), lambda: work(owner), opening_s=OPEN_S) async def transaction[T](conn: Any, work: Callable[[], Awaitable[T]]) -> T: diff --git a/tests/integration/test_cell_control_cleanup.py b/tests/integration/test_cell_control_cleanup.py index e057fef323..8f68939b62 100644 --- a/tests/integration/test_cell_control_cleanup.py +++ b/tests/integration/test_cell_control_cleanup.py @@ -105,6 +105,8 @@ async def test_native_prepare_second_cancel_preserves_first_and_releases_locks( real_close = asyncpg.Connection.close observed, close_started, close_return = asyncio.Event(), asyncio.Event(), asyncio.Event() backend = {} + close_interrupted = asyncio.Event() + monkeypatch.setattr(cell_control, "TRANSACTION_S", 0.4) async def read(conn, **_): backend["conn"], backend["pid"] = conn, conn.get_server_pid() @@ -115,7 +117,11 @@ async def delayed_close(self, **kwargs): await real_close(self, **kwargs) if self is backend.get("conn"): close_started.set() - await close_return.wait() + try: + await close_return.wait() + except asyncio.CancelledError: + close_interrupted.set() + raise monkeypatch.setattr(cell_prepare, "_read_rows", read) monkeypatch.setattr(asyncpg.Connection, "close", delayed_close) @@ -128,6 +134,10 @@ async def delayed_close(self, **kwargs): task.cancel("first native cancellation") await asyncio.wait_for(close_started.wait(), 1) task.cancel("second cancellation") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not close_interrupted.is_set(), "second signal cannot interrupt public close" + assert not task.done() with pytest.raises(asyncio.CancelledError, match="first native cancellation"): await asyncio.wait_for(task, 0.5) await released(native_database, backend["pid"]) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index 11bf1bf094..a018784d21 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -8,7 +8,6 @@ import pytest from server.cli import cell_attach, cell_control -from server.core import pg from tests.unit.cell_control_owner import owner from tests.unit.test_cell_control import snapshot @@ -52,6 +51,9 @@ async def close(self, **kwargs): def terminate(self): events.append("abort") + def is_closed(self): + return False + async def work(): entered.set() try: @@ -78,7 +80,7 @@ async def work(): @pytest.mark.asyncio -async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(monkeypatch): +async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() conn.terminate = lambda: None @@ -88,8 +90,8 @@ async def close(**_): await asyncio.Future() conn.close.side_effect = close - owned = owner(conn) - monkeypatch.setattr(pg, "_RELEASE_TIMEOUT_S", 0.03) + owned = cell_control.Owner(exit_s=0.03) + owned.conn = conn started = time.monotonic() with pytest.raises(TimeoutError): await owned.close() diff --git a/tests/unit/test_cell_control_abort.py b/tests/unit/test_cell_control_abort.py new file mode 100644 index 0000000000..890ddcba27 --- /dev/null +++ b/tests/unit/test_cell_control_abort.py @@ -0,0 +1,185 @@ +"""A failed fallback abort cannot replace the native owner's first failure.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control + +pytestmark = pytest.mark.asyncio +ERRORS = (ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) +ABORT_ERRORS = (RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt) + + +async def outcome(work): + try: + result = await cell_control._captured(work) + except BaseException as error: # noqa: BLE001 — observe the identical native failure inertly. + return error + assert isinstance(result, cell_control._Fatal), "a failed required close cannot succeed" + return result.error + + +@pytest.mark.parametrize("first_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_direct_owner_keeps_first_close_failure_and_retrieves_child( + monkeypatch, first_kind, abort_kind +): + first, secondary = first_kind("first close failure"), abort_kind("abort failure") + aborted = [] + + class Connection: + def terminate(self): + aborted.append(True) + raise secondary + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + close = AsyncMock(side_effect=first) + monkeypatch.setattr(cell_control.pg, "close_owned", close) + result = await outcome(owner.close) + assert result is first + assert aborted == [True] + close.assert_awaited_once() + assert owner.closed and owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +@pytest.mark.parametrize("first_kind", ERRORS) +async def test_outer_signal_wins_over_later_close_and_abort_failures( + monkeypatch, first_kind, abort_kind +): + entered, finish = asyncio.Event(), asyncio.Event() + first, secondary = first_kind("close failed after signal"), abort_kind("abort failure") + seen = [] + actual_wait = asyncio.wait + owner = cell_control.Owner(exit_s=0.2) + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + await finish.wait() + raise first + + async def wait(*args, **kwargs): + try: + return await actual_wait(*args, **kwargs) + except asyncio.CancelledError as error: + seen.append(error) + raise + + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + monkeypatch.setattr(cell_control.asyncio, "wait", wait) + task = asyncio.create_task(outcome(owner.close)) + try: + await entered.wait() + original_end = owner.close_end + task.cancel("original outer cancellation") + await asyncio.sleep(0) + assert len(seen) == 1 and not task.done() + finish.set() + result = await task + assert result is seen[0] + assert owner.close_end == original_end + assert owner._close_task.done() + assert owner._close_task.cancelled() or not owner._close_task._log_traceback + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_abort_failure_still_cancels_and_retrieves_expired_close_child( + monkeypatch, abort_kind +): + entered, ended = asyncio.Event(), asyncio.Event() + secondary = abort_kind("abort failure") + + class Connection: + def terminate(self): + raise secondary + + async def close(conn, *, end): + entered.set() + try: + await asyncio.Future() + finally: + ended.set() + raise ValueError("late close failure") + + owner = cell_control.Owner(exit_s=0.02) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert isinstance(result, TimeoutError) + assert time.monotonic() - started < 0.2 + assert entered.is_set() + await asyncio.wait_for(ended.wait(), 0.2) + await asyncio.sleep(0) + assert owner._close_task.done() and not owner._close_task._log_traceback + + +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_self_cancelled_close_is_terminal_even_when_abort_fails(monkeypatch, abort_kind): + first = asyncio.CancelledError("close cancelled itself") + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failure") + + async def close(conn, *, end): + calls.append("close") + raise first + + owner = cell_control.Owner(exit_s=60) + owner.conn = Connection() + monkeypatch.setattr(cell_control.pg, "close_owned", close) + started = time.monotonic() + result = await outcome(owner.close) + assert result is first + assert time.monotonic() - started < 0.2 + assert calls == ["close", "abort"] + + +@pytest.mark.parametrize("wait_kind", ERRORS) +@pytest.mark.parametrize("abort_kind", ABORT_ERRORS) +async def test_failed_reap_abort_cannot_replace_the_callers_first_failure( + monkeypatch, wait_kind, abort_kind +): + owner = cell_control.Owner() + owner.close_end = time.monotonic() + 0.2 + calls = [] + + class Connection: + def terminate(self): + calls.append("abort") + raise abort_kind("abort failed while reaping") + + async def failed_wait(*args, **kwargs): + raise wait_kind("reap wait interrupted") + + owner.conn = Connection() + child = owner._start(lambda: asyncio.Future()) + monkeypatch.setattr(cell_control.asyncio, "wait", failed_wait) + try: + try: + result = await cell_control._captured(lambda: owner._stop({child})) + except BaseException as error: # noqa: BLE001 — the probe must expose secondary native errors. + result = error + assert result is None, "reaping cannot replace an earlier operation failure" + assert calls == ["abort"] + finally: + child.cancel() + await asyncio.gather(child, return_exceptions=True) diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py new file mode 100644 index 0000000000..2cce1d25cf --- /dev/null +++ b/tests/unit/test_cell_place_owner.py @@ -0,0 +1,233 @@ +"""Placement reuses the control owner without replacing its phase budgets.""" + +from __future__ import annotations + +import asyncio +import time +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control +from server.core import pg +from tests.unit.test_cell_attach import FakeConn + +pytestmark = pytest.mark.asyncio + + +async def test_custom_exit_is_used_only_outside_a_local_phase(): + owner = cell_control.Owner(exit_s=0.09) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + before = time.monotonic() + assert await owner.run(opening, AsyncMock(return_value=17), opening_s=0.3) == 17 + assert before + 0.09 <= owner.close_end <= time.monotonic() + 0.09 + assert conn.executed == ["CLOSE"] + + +@pytest.mark.parametrize("occupied", ["conn", "phase", "task", "closed", "repeated"]) +async def test_command_entry_refuses_reuse_before_any_callback(occupied): + owner = cell_control.Owner() + held = None + if occupied == "conn": + owner.conn = FakeConn() + elif occupied == "phase": + owner.phase_end = time.monotonic() + 1 + elif occupied == "task": + held = owner._start(lambda: asyncio.Future()) + elif occupied == "closed": + await owner.close() + else: + await owner.run(AsyncMock(), AsyncMock(), opening_s=0.1) + opening, body = AsyncMock(), AsyncMock() + try: + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(opening, body, opening_s=0.1) + opening.assert_not_called() + body.assert_not_called() + finally: + if held is not None: + held.cancel() + await asyncio.gather(held, return_exceptions=True) + + +@pytest.mark.parametrize("stage", ["opening", "body"]) +async def test_recursive_entry_cannot_replace_the_original_backend(stage): + owner = cell_control.Owner() + original = FakeConn() + other_open, other_body = AsyncMock(), AsyncMock() + + async def recurse(): + with pytest.raises(cell_control.Refused, match="fresh"): + await owner.run(other_open, other_body, opening_s=0.1) + + async def opening(): + if stage == "opening": + await recurse() + owner.conn = original + + async def body(): + if stage == "body": + await recurse() + return 3 + + assert await owner.run(opening, body, opening_s=0.1) == 3 + other_open.assert_not_called() + other_body.assert_not_called() + assert owner.conn is original and original.executed == ["CLOSE"] + + +async def test_native_wrapper_keeps_its_existing_three_budgets(monkeypatch): + observed = {} + actual_run = cell_control.Owner.run + actual_call = cell_control.Owner.call + + async def wrapped_run(self, opening, work, *, opening_s): + observed.update(opening=opening_s, exit=self.exit_s) + return await actual_run(self, opening, work, opening_s=opening_s) + + async def wrapped_call(self, work, *, total_s): + observed.setdefault("phases", []).append(total_s) + return await actual_call(self, work, total_s=total_s) + + async def opening(self, what): + assert what == "native" + self.conn = FakeConn() + + monkeypatch.setattr(cell_control.Owner, "run", wrapped_run) + monkeypatch.setattr(cell_control.Owner, "call", wrapped_call) + monkeypatch.setattr(cell_control.Owner, "_open", opening) + assert ( + await cell_control.run("native", lambda owner: owner.transaction(AsyncMock(return_value=7))) + == 7 + ) + assert observed == {"opening": 75.0, "exit": pg._RELEASE_TIMEOUT_S, "phases": [75.0, 60.0]} + + +async def test_failed_phase_keeps_its_end_instead_of_custom_exit(): + owner = cell_control.Owner(exit_s=99) + conn = FakeConn() + + async def opening(): + owner.conn = conn + + async def body(): + return await owner.call(lambda: asyncio.Future(), total_s=0.04) + + before = time.monotonic() + with pytest.raises(TimeoutError): + await owner.run(opening, body, opening_s=0.1) + assert before < owner.close_end <= before + 0.1 + assert owner.close_end == owner.phase_end + assert conn.executed == ["CLOSE"] + + +async def test_public_close_defers_repeated_signals_until_acknowledged_within_original_end(): + events = [] + closing, finish = asyncio.Event(), asyncio.Event() + + class Connection: + async def close(self, **kwargs): + events.append("close") + closing.set() + await finish.wait() + events.append("ack") + + def terminate(self): + events.append("abort") + + def is_closed(self): + return "ack" in events + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + original_end = owner.close_end + task.cancel("first signal") + await asyncio.sleep(0) + task.cancel("second signal") + await asyncio.sleep(0) + assert not task.done(), "a second signal must not interrupt owned public close" + assert owner.close_end == original_end + finish.set() + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert events == ["close", "ack"] + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + + +async def test_repeated_signals_cannot_extend_an_expired_public_close(): + closing = asyncio.Event() + conn = FakeConn() + + async def stalled(**kwargs): + closing.set() + await asyncio.Future() + + conn.close = stalled + owner = cell_control.Owner(exit_s=0.04) + owner.conn = conn + started = time.monotonic() + task = asyncio.create_task(owner.close()) + await closing.wait() + end = owner.close_end + + async def signals(): + while not task.done(): + task.cancel("first signal") + await asyncio.sleep(0.001) + + signal_task = asyncio.create_task(signals()) + try: + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "repeated signals cannot restart the original cleanup allowance" + with pytest.raises(asyncio.CancelledError, match="first signal"): + await task + assert time.monotonic() - started < 0.2 + assert owner.close_end == end + assert "TERMINATE" in conn.executed + finally: + signal_task.cancel() + await asyncio.gather(signal_task, task, return_exceptions=True) + + +async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): + closing, acknowledged = asyncio.Event(), asyncio.Event() + events = [] + + class Connection: + async def close(self, **kwargs): + closing.set() + await acknowledged.wait() + events.append("ack") + + def is_closed(self): + # An aborted transport is closed even while its cancellation is pending. + return True + + def terminate(self): + events.append("abort") + + owner = cell_control.Owner(exit_s=0.2) + owner.conn = Connection() + task = asyncio.create_task(owner.close()) + try: + await closing.wait() + task.cancel("stop") + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() and events == [] + acknowledged.set() + with pytest.raises(asyncio.CancelledError, match="stop"): + await task + assert events == ["ack"] + finally: + acknowledged.set() + await asyncio.gather(task, return_exceptions=True) From f378a281cc5095a50b3ac08adff6138ed4e69f14 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 01:50:13 +0000 Subject: [PATCH 077/181] Match persisted vector precision and preserve attach cleanup errors Compare expected vectors at their BFloat16 storage precision. Keep the first attach failure when fallback abort or a cleanup diagnostic also fails. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 29 +++-- server/cli/cell_attach_pg.py | 10 +- tests/unit/test_bench_cell_trigger_gate.py | 107 ++++++++++++++++++ tests/unit/test_cell_attach_abort.py | 119 +++++++++++++++++++++ 4 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_cell_attach_abort.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index b1874e1fd9..17c0230ef7 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -18,6 +18,7 @@ import socket import stat import statistics +import struct import subprocess import sys import time @@ -1035,20 +1036,31 @@ def _normal(value: Any) -> Any: return value +def _stored_vector(value: Any) -> list[float]: + _require( + isinstance(value, list) + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in value), + "finite vector", + ) + try: + # The insert driver stores the high 16 bits of each Float32 component. + return [struct.unpack(" None: for field_name in fields: left, right = expected.get(field_name), actual.get(field_name) if field_name in ("embedding", "memory_key_embedding"): + left = _stored_vector(left) _require( - isinstance(left, list) - and isinstance(right, list) + isinstance(right, list) and len(left) == len(right) - and all( - math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) - for a, b in zip(left, right, strict=True) - ), + and all(isinstance(x, (int, float)) and math.isfinite(x) for x in right) + and left == right, phase + " vector", ) else: @@ -1350,7 +1362,10 @@ async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> except BaseException: if prior is None: raise - artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + try: + artifact.emit("connection_cleanup_error", error_type=type(sys.exception()).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace restoration failure. + pass async def _restore(triggers: Triggers, conn: Any, end: float) -> None: diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index ca6fedf794..271631265f 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -129,7 +129,10 @@ async def supervise[T](conn: Any, work: Coroutine[Any, Any, T], *, end: float) - try: await asyncio.wait({task}, timeout=max(0.0, end - time.monotonic())) except BaseException: # noqa: BLE001 — repeated cancellation preserves the first failure. - pg._terminate(conn) + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the work failure. + pass raise @@ -169,9 +172,12 @@ async def close_context( if isinstance(result := task.result(), _Interrupted): raise result.error except BaseException as caught: # noqa: BLE001 — exit cannot replace the first failure. - pg._terminate(conn) if failure is None: failure = caught + try: + pg._terminate(conn) + except BaseException: # noqa: BLE001 — abort cannot replace the selected failure. + pass finally: if not task.done(): task.cancel() diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 29dcce134b..824808214a 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -461,6 +461,52 @@ def test_stored_payload_vector_version_and_audit_mismatch_refuse(field, expected gate._compare({field: expected}, {field: actual}, [field], "stored") +def stored_bfloat16(vector): + """Exercise the installed insert writer and query reader without a server.""" + import array + + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.datatypes.numeric import BFloat16 + + class Encoded: + def __init__(self, raw): + self.raw = raw + + def read_array(self, kind, count): + result = array.array(kind) + result.frombytes(self.raw) + assert len(result) == count + return result + + kind = BFloat16(TypeDef()) + raw = bytearray() + kind._write_column_binary(vector, raw, SimpleNamespace(column_name="embedding")) + return kind._read_column_binary( + Encoded(raw), len(vector), SimpleNamespace(use_numpy=False), None + ) + + +@pytest.mark.parametrize("field", ["embedding", "memory_key_embedding"]) +def test_storage_vector_matches_installed_bfloat16_and_rejects_corrupted_neighbor(field): + from server.embedding.fake import FakeNGramEmbedder + + vector = FakeNGramEmbedder(dim=1024)._embed_raw(gate._payload("gate-review-owned-1")) + stored = stored_bfloat16(vector) + assert sum(a != b for a, b in zip(vector, stored, strict=True)) > 0 + gate._compare({field: vector}, {field: stored}, [field], "stored") + import struct + + corrupted = list(stored) + index = next(i for i, value in enumerate(stored) if value > 0) + bits = struct.unpack(" Date: Tue, 17 Nov 2026 02:10:19 +0000 Subject: [PATCH 078/181] WIP: coordinate cell configuration updates and address proofs Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 4 + docs/architecture/cells.md | 71 ++-- docs/operations/cell-update.md | 143 +++++++ docs/reference/cli.md | 1 + llms.txt | 1 + server/cli/_ch.py | 22 + server/cli/cell.py | 2 + server/cli/cell_prepare.py | 19 +- server/cli/cell_update.py | 120 ++++++ server/cli/cell_update_addresses.py | 64 +++ server/cli/cell_update_participants.py | 256 ++++++++++++ server/cli/cell_update_pg.py | 128 ++++++ server/cli/cell_update_probe.py | 44 ++ server/cli/cell_update_run.py | 229 +++++++++++ server/router/app.py | 110 ++++- server/router/directory.py | 80 ++++ .../integration/test_cell_update_addresses.py | 172 ++++++++ .../integration/test_cell_update_directory.py | 103 +++++ .../test_cell_update_participants_ch.py | 142 +++++++ tests/integration/test_cell_update_run_pg.py | 384 ++++++++++++++++++ tests/unit/snapshots/cli_help.json | 9 +- tests/unit/snapshots/cli_tree.json | 175 +++++++- tests/unit/test_cell_cli_orgs.py | 1 + tests/unit/test_cell_control_census.py | 35 ++ tests/unit/test_cell_update.py | 96 +++++ tests/unit/test_cell_update_participants.py | 301 ++++++++++++++ tests/unit/test_cell_update_probe.py | 74 ++++ tests/unit/test_cell_update_run.py | 231 +++++++++++ 28 files changed, 2980 insertions(+), 37 deletions(-) create mode 100644 docs/operations/cell-update.md create mode 100644 server/cli/cell_update.py create mode 100644 server/cli/cell_update_addresses.py create mode 100644 server/cli/cell_update_participants.py create mode 100644 server/cli/cell_update_pg.py create mode 100644 server/cli/cell_update_probe.py create mode 100644 server/cli/cell_update_run.py create mode 100644 tests/integration/test_cell_update_addresses.py create mode 100644 tests/integration/test_cell_update_participants_ch.py create mode 100644 tests/integration/test_cell_update_run_pg.py create mode 100644 tests/unit/test_cell_update.py create mode 100644 tests/unit/test_cell_update_participants.py create mode 100644 tests/unit/test_cell_update_probe.py create mode 100644 tests/unit/test_cell_update_run.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ed512176d..60de7d1b74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,6 +902,10 @@ jobs: tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py tests/integration/test_cell_place.py + tests/integration/test_cell_update_directory.py + tests/integration/test_cell_update_addresses.py + tests/integration/test_cell_update_run_pg.py + tests/integration/test_cell_update_participants_ch.py tests/integration/test_cell_place_cleanup.py tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 0234887370..77d90d4d40 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -418,8 +418,12 @@ replaces a later cell's under the directory lock after the same check and a fres echo. The first cell's probe and upstream addresses are copied from the router's configuration at registration, and `loom cell update` replaces them from it, once every router replica runs the new values, after the same probe and a check that the new -upstream reports the cell's deployment id. A prepare rerun never moves an address: it -refuses while the router replica it reaches is configured with others. +upstream reports the cell's deployment id. The generation/address CAS does not prove +all-router agreement: an old router can read fresh expected values and reverse the +addresses. Keep old endpoints reachable until every router refreshes its routing table +or restarts after the directory write; config rollout alone does not flush the cache, +which can persist indefinitely during directory failure. A prepare rerun never moves +an address: it refuses while the router replica it reaches is configured with others. **The `first` row.** Prepare writes an explicit, immutable `first` row into every cell's Postgres (`loom.cell_first`: a row trigger refuses its update and delete and a statement @@ -526,27 +530,39 @@ directory, both values of `loom.cell_self`. A prepared cell's issuers and org pa are reserved nowhere: the router routes no issuer to it and takes no placement from it until attach has run its overlap refusals and written its directory row. -A change to what a cell accepts goes through `loom cell update --config `, under -the same directory lock as attach; it computes `next` from that configuration with the -same function the processes use, and refuses while a different `next` is pending; rerun -with the same configuration, or `--finish`, first writes `next` wherever it is missing. -It reruns attach's overlap refusals against every other cell's directory row, counting -both values of each, writes the new fingerprint as `next` to the directory and then to -`loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. Finalization first narrows the cell's accepted value durably while the directory -keeps both reservations. It then requires a complete post-cutoff participant census, +A change to what a cell accepts goes through `loom cell update --config ` +([operator procedure](../operations/cell-update.md)). The nonsecret settings file +overlays the CLI's effective settings without installing them. It computes `next` +with the same function the processes use and refuses while a different `next` is +pending. Under the directory lock, overlap checks count both values of every other +cell. Configured placements, their counts, issuer routing and the directory pair +commit together before local tenancy repair and acceptance of the pair. + +`--finish` selects next; `--revert` selects current. Each repairs local tenancy from +actual same-cell immutable placement winners for both reserved values, using an +operator read fenced by the generation, exact registration and pair, locked +root-secret fingerprint, audit and synchronous WAL acknowledgement. First-cell +lookup fallback is not evidence of a placement. The attaching first cell may still +lack placements for its preexisting configured orgs; other missing placements +require directory metadata recovery. + +Finalization durably narrows local acceptance while the directory keeps both +reservations. It then takes a complete post-cutoff participant census and requires permanent guarded retirement or verified exact death for every captured identity, -and a serving target replacement outside that captured set. Heartbeat expiry and -request deadlines do not prove this. Only afterward may the directory release its -old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) -and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or -pattern is reserved in the directory before any process accepts it, and a narrowed one -is released only after none does. A refusal or a wait that runs out keeps both values -and prints the processes not yet reporting `next`; rerunning resumes, and -`loom cell update --revert` waits the same way until no live process matches only `next` -(refusing when `--wait` runs out), then drops `next`, in `loom.cell_self` first, under -the same lock; a configured org the update placed stays placed in the cell (a placement -is never deleted). +including target processes, and a serving target replacement outside that set. +Only then may the directory release its old reservation. Heartbeat expiry, +request deadlines, missing participant rows and workers-only replacements do not +prove this. Online finalization needs the +[lifecycle baseline](#participant-lifecycle-baseline) and controller fencing; +SCIM-on candidates and finish remain refused until per-cell SCIM is implemented. +Revert to SCIM-off current remains supported. + +A timeout after cutoff keeps local acceptance single and the directory dual; the +command reports incomplete turnover. Repeating the selected action takes a fresh +census. An identical `--config` repairs directory-first interruption and can widen +local acceptance again; subsequent finalization takes another cutoff and census. +A singleton no-op still proves local synchronous durability. Every configured org +placed by an update stays placed and retains its tenancy row after revert. A new SCIM token changes no fingerprint and needs no `loom cell update`. A cell changes its identity provider application as a single deployment does ([SSO and @@ -1883,10 +1899,10 @@ An integration test with two cells and a router in CI's Postgres job: value, a winner of this cell's name under another deployment id answers 503 and writes no tenancy row, and a process whose configuration differs from its cell's row places nothing; -- a cell attached with SCIM off and switched on by `loom cell update` refuses while an - agent-signup token is live, one minted while the update rolls out included, and a - process configured with another cell's issuer answers 503 to every bearer request, - reached through the router or directly; +- `loom cell update --config` and `--finish` refuse a SCIM-on target before an + acceptance write until per-cell SCIM support exists; reverting a SCIM-on pending + value to SCIM-off current remains supported; a process configured with another + cell's issuer answers 503 to every bearer request, through the router or directly; - during a `loom cell update`, processes on the old and the new configuration both serve, `--finish` releases a narrowed issuer only after no process accepts it, an update that dies after its directory write keeps the widened issuer reserved, and @@ -1900,7 +1916,8 @@ An integration test with two cells and a router in CI's Postgres job: refused, and a cell that starts accepting another cell's issuer takes no placements; - a process matching neither value of its row answers 503 to sign-in, refresh, the OAuth token and device endpoints, the sign-in callback and signup, and its probe route still - echoes prepare's nonce; `--revert` waits for processes matching only `next`; + echoes prepare's nonce; `--revert` requires permanent retirement of every captured + participant, including ones already on current; `update --config` refuses while a different `next` is pending, and an update that died before its `loom.cell_self` write resumes forward; prepare refuses while a pre-N process is live; a configuration change while `attaching` goes through diff --git a/docs/operations/cell-update.md b/docs/operations/cell-update.md new file mode 100644 index 0000000000..1a185b61df --- /dev/null +++ b/docs/operations/cell-update.md @@ -0,0 +1,143 @@ +# Update a cell's configuration or addresses + +`loom cell update` reserves a configuration before the cell accepts it. Run it with +that cell's stores, `LOOM_CELL`, and the router operator credential configured. It +holds the cell's ATTACH lock on one direct Postgres session. Prepare, attach, +restore, another update, and the documented state cleanup cannot run concurrently. +An interrupted command exits unsuccessfully; rerun the same action after correcting +the reported cause. + +## Reserve a configuration + +Write a nonsecret settings file: + +```yaml +version: 1 +settings: + tracker_org: operations +``` + +```bash +loom cell update --cell first --config target.yaml +``` + +The file overlays the CLI's effective settings. Every supplied setting is validated, +including values outside the fingerprint. Secret fields are refused. The command +neither installs the file nor changes the running processes' configuration. It +prints fingerprint digests and changed field names. + +The deployment id, registered public and OAuth host, and bucket and prefix cannot +change. The root-secret fingerprint must match the directory's locked value. A +candidate with SCIM enabled refuses; per-cell SCIM support is required before such +a rollout. Reverting a pending SCIM-on candidate to a recorded SCIM-off current +value remains possible. + +After attach has begun, the directory reserves both current and next values and +places newly configured orgs before the cell's Postgres accepts next. Overlap checks +include both values of every other attached or attaching cell. A different pending +value refuses. Repeating the identical candidate repairs an interrupted local +write without placing an org twice. + +New configured placements obey the cell's closed flag and quota. Existing winners +in that cell remain valid while closed or full. An org held elsewhere refuses the +whole reservation. A later cell still attaching refuses configured-org changes; +a first cell attaching after begin accepts them through the normal placement gate. + +Before attach/begin has recorded the directory fingerprint, update claims the +registration's generation and changes only the local pair. It places no org and +reserves no issuer or pattern. Attach checks and reserves both local values later. + +## Finish or revert + +Install and roll out the reserved configuration, then run: + +```bash +loom cell update --cell first --finish --wait 600 +``` + +To keep current instead: + +```bash +loom cell update --cell first --revert --wait 600 +``` + +These forms first repair local tenancy for actual same-cell configured placements +from both directory values. Their operator read compares the exact generation, +registration, pair, and root-secret fingerprint, writes an audit and WAL fence, +and waits for synchronous commit. Public lookup's first-cell fallback is not proof +that an immutable placement exists. A missing placement outside the attaching +first cell requires directory metadata recovery. + +Finalization durably narrows local acceptance first. The directory keeps both +reservations until every participant captured by a complete post-cutoff census +has permanently retired or its exact process is proved dead with no outstanding +ClickHouse query. Captured target processes also turn over. A target-serving +replacement must join a live instance, a marked cell heartbeat, and a guarded +participant outside the captured set. A workers-only process cannot satisfy this. + +Every configured ClickHouse replica must answer. Replicated participant tables +synchronize after the local cutoff; the census unions their newest rows without +age filters, sampling, caches, or partial-result limits. Heartbeat expiry, +`drained`, a missing row, an empty query list alone, a request deadline, or deletion +of a pod object does not prove permanent retirement. An uninspectable process or +unreadable topology refuses finalization. + +Online finalization requires the [participant lifecycle baseline](../architecture/cells.md#participant-lifecycle-baseline) +and [controller fencing](../architecture/cells.md#control-mutation-fencing). Positively +terminate or fence every legacy server, worker, one-off command and control writer, +including paused boots, before using this protocol. Prevent old binaries from +returning. Inventory actual process/container termination or independent node +fencing; an expired heartbeat or missing API object is insufficient. The CLI +cannot establish this external inventory from its census. Unknown legacy identities +remain blockers. No command flag asserts or bypasses this prerequisite. + +`--wait` is finite and between 0 and 86400 seconds. Zero still makes a bounded +initial attempt. Local transactions and complete census reads have their own +bounded attempts. If turnover times out, local acceptance stays single and the +directory stays dual. Keep the target replacement serving, resolve the named +participants and rerun the same action. Each rerun takes a fresh census. Retrying +`--config` can widen local acceptance again and requires another cutoff and census +before finalization. All directory placements and local tenancy rows remain after +revert, including orgs introduced by the pending value. + +| Interruption | Recorded state | Recovery | +| -------------------------------------------------------------- | ----------------------------------------- | ---------------------------------------------------- | +| Directory reservation commits before local write | Directory dual; local pair may be old | Repeat config, finish, or revert explicitly | +| Local cutoff commits before turnover or final directory commit | Directory dual; local single | Repeat the selected finish or revert | +| Generation or ATTACH backend changes | An in-flight directory reply may be stale | Stop; a fresh invocation claims again and recomputes | +| Final directory singleton already matches local | Both single | The no-op still proves synchronous local durability | + +## Refresh addresses + +For a later cell: + +```bash +loom cell update --cell second --probe-url https://second-probe.example.test +``` + +For the first cell, roll out the intended `LOOM_ROUTER_UPSTREAM_URL` and +`LOOM_ROUTER_UPSTREAM_PROBE_URL` to **every router first**, then run without an +action flag: + +```bash +loom cell update --cell first +``` + +The contacted router supplies its normalized first-cell addresses. The probe must +echo fresh Postgres, ClickHouse, and configured bucket nonces and the registered +deployment id, bucket and prefix. A main listener cannot substitute for the probe. +The first cell's main upstream must independently report that deployment id. The +router rechecks its configured addresses at commit. A later refresh changes only +its probe URL; it cannot move the registered host, stores or upstream. + +A generation and address comparison excludes stale requests; it does not establish +fleet-wide configuration agreement. An old router can read the current expected +row and reverse the addresses if it still runs the old configuration. Keep the +old endpoints reachable until every router demonstrably refreshes its routing +table or restarts **after** the directory change. A configuration rollout alone +does not flush cached routes. During a directory outage a router can keep its last +routing table indefinitely. The command cannot certify all-router agreement. + +Configuration actions, finish, revert and probe refresh are mutually exclusive. +Probe readiness retries use `--wait`; invalid proofs refuse immediately. Nonce +cleanup compares this run's values, leaving another run's nonce untouched. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1ab8b69510..3b0ea68103 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -335,6 +335,7 @@ python scripts/codegen/docs_reference.py | `loom cell place` | Place an org in this cell and record its tenancy. | `--cell`, `--org`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | | `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | +| `loom cell update` | Reserve configuration, finish or revert its rollout, or refresh verified addresses. | `--cell`, `--config`, `--finish`, `--revert`, `--probe-url`, `--router`, `--wait` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/llms.txt b/llms.txt index 488d65bdb9..4fb3cb990c 100644 --- a/llms.txt +++ b/llms.txt @@ -106,6 +106,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Backup and recovery](docs/operations/backup-and-recovery.md): A Loom backup is a ClickHouse backup, plus the document bucket while any catalog row names an object in it, and under the Postgres authority a backup of Postgres too. - [Attaching the first cell](docs/operations/cell-attach.md): `loom cell attach` turns a prepared first cell (`loom cell prepare`, mode `recording`, directory row `attaching`) into a `serving` one: it places every org the cell holds in the router's directory, raises the cell's mode to `enforced` and… - [Cell control upgrades and recovery](docs/operations/cell-control.md): Cell control mutations require generation-aware clients and router handlers, and direct access to the cell's Postgres. +- [Update a cell's configuration or addresses](docs/operations/cell-update.md): `loom cell update` reserves a configuration before the cell accepts it. - [ClickHouse Cloud and Managed Postgres](docs/operations/clickhouse-cloud.md): This guide runs Loom on Kubernetes over two managed services: ClickHouse Cloud for projections, vectors, history and analytics, and ClickHouse Managed Postgres as the memory authority. - [Code index operations](docs/operations/code-index-operations.md): This page covers how to enable the code index workers, configure repositories and keep the index current; the code index runs only when an operator enables its workers and configures repositories. - [Deployment](docs/operations/deployment.md): Loom is one server image plus ClickHouse (and Postgres, under `LOOM_STORAGE_AUTHORITY=postgres`), with the document extractor service as an image of its own (container images). diff --git a/server/cli/_ch.py b/server/cli/_ch.py index c67479b965..70173eff64 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -394,3 +394,25 @@ async def _go() -> T: await disconnect(ch) return asyncio.run(_go()) + + +def run_control_connected[T](run: Callable[[CHClient], Awaitable[T]], *, org_free: OrgFree) -> T: + """Connect an operator-only coordinator without admitting model or org work. + + Such a coordinator cannot join the participant set whose retirement it waits + for. It still verifies paired stores and the explicit org-free write gate. + """ + from server.core.ch import get_client + + _declared(None, org_free) + + async def go() -> T: + ch = get_client() + await connect_verified(ch) + try: + await check_acceptance(org_free=org_free) + return await run(ch) + finally: + await disconnect(ch) + + return asyncio.run(go()) diff --git a/server/cli/cell.py b/server/cli/cell.py index a6fb4e21e5..4511fb5220 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -465,7 +465,9 @@ def _add_cell_parser(sub: Any) -> None: from server.cli.cell_attach import add_attach_parser from server.cli.cell_place import add_place_parser from server.cli.cell_prepare import add_prepare_parser + from server.cli.cell_update import add_update_parser add_place_parser(csub) add_prepare_parser(csub) add_attach_parser(csub) + add_update_parser(csub) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..82b0d1a1b0 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -295,11 +295,14 @@ async def _register( router: httpx.URL, body: dict[str, Any], transport: httpx.AsyncBaseTransport | None, + *, + path: str = "/register", + command: str = "prepare", ) -> dict[str, Any]: """The router's 200 answer, retrying a busy or unreachable probe listener within `--wait`; Refused otherwise.""" token = settings.router_operator_token - url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") + url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}{path}") deadline = time.monotonic() + args.wait async with httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport @@ -310,12 +313,18 @@ async def _register( except httpx.HTTPError as e: # A rerun answers the row a lost answer may have written. raise Refused( - f"no answer from the router ({type(e).__name__}): run prepare again" + f"no answer from the router ({type(e).__name__}): run {command} again" ) from None answer = cell_cli.router_answer(r, {"cell": args.cell}, cell_cli.OPERATOR_REFUSALS) + if ( + command == "update" + and r.status_code == 404 + and (answer is None or answer.get("code") != "NO_SUCH_CELL") + ): + raise Refused("a router replica is not upgraded for cell update") if answer is None: raise Refused( - f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" + f"{cell_cli.not_the_router(r)}; run {command} again once the router answers" ) if r.status_code == 200: return answer @@ -323,7 +332,7 @@ async def _register( if code == "BACKEND_UNAVAILABLE": raise Refused( f"the directory did not answer ({answer['error']}); the registration may " - "have landed: run prepare again" + f"have landed: run {command} again" ) if code not in ("PROBE_BUSY", "PROBE_UNREACHABLE"): raise Refused(f"the router refused ({r.status_code}): {answer['error']}") @@ -337,7 +346,7 @@ async def _register( raise Refused( f"the router could not probe the cell within --wait: {answer['error']}" ) - print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) + print(f"loom cell {command}: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) diff --git a/server/cli/cell_update.py b/server/cli/cell_update.py new file mode 100644 index 0000000000..9e6b52d8f8 --- /dev/null +++ b/server/cli/cell_update.py @@ -0,0 +1,120 @@ +"""Operator configuration updates and verified address refresh for a prepared cell.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import sys +from pathlib import Path +from typing import Any + +from server.cli import cell as cell_cli +from server.cli._ch import run_control_connected, writes +from server.cli.cell_prepare import _WAIT_MAX_S +from server.core import cell +from server.settings import settings + + +def _args_problem(args: argparse.Namespace) -> str: + if not cell.NAME.fullmatch(args.cell or ""): + return "--cell names 1 to 32 of a-z, 0-9, -" + if settings.cell != args.cell: + return f"update runs with this cell's configuration: set LOOM_CELL={args.cell}" + if settings.storage_authority != "postgres" or not settings.pg_dsn: + return "update needs LOOM_STORAGE_AUTHORITY=postgres and this cell's LOOM_PG_DSN" + if not math.isfinite(args.wait) or not 0 <= args.wait <= _WAIT_MAX_S: + return f"--wait is finite, nonnegative and at most {_WAIT_MAX_S} seconds" + if sum(bool(value) for value in (args.config, args.finish, args.revert, args.probe_url)) > 1: + return "choose one of --config, --finish, --revert or --probe-url" + return "" + + +def cmd_update(args: argparse.Namespace) -> int: + if reason := _args_problem(args): + print(f"loom cell update: {reason}", file=sys.stderr) + return 2 + router = cell_cli.operator_router(args) + if isinstance(router, str): + print(f"loom cell update: {router}", file=sys.stderr) + return 2 + + async def run(ch: Any) -> int: + from server.cli.cell_update_run import update + + try: + answer = await update(ch, args, router) + except asyncio.CancelledError: + print( + "loom cell update: interrupted; outcome may be unknown; rerun the same action", + file=sys.stderr, + ) + raise + except Exception as error: # noqa: BLE001 — failures must identify the bounded recovery action. + print( + f"loom cell update: stopped ({type(error).__name__}: {error}); " + "rerun the same action", + file=sys.stderr, + ) + return 1 + print(json.dumps(answer, indent=2, sort_keys=True)) + return 0 + + return run_control_connected( + run, org_free=writes("updates this cell's reserved configuration and verified addresses") + ) + + +def add_update_parser(csub: Any) -> None: + parser = csub.add_parser( + "update", + help="Reserve configuration, finish or revert its rollout, or refresh verified addresses.", + ) + parser.add_argument("--cell", required=True, help="This cell's name (LOOM_CELL)") + action = parser.add_mutually_exclusive_group() + action.add_argument( + "--config", + type=Path, + help=( + "Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; " + "does not install the file" + ), + ) + action.add_argument( + "--finish", + action="store_true", + help="Promote next after durable local narrowing and permanent participant turnover", + ) + action.add_argument( + "--revert", + action="store_true", + help=( + "Retain current after durable local narrowing and permanent participant " + "turnover; keep placements" + ), + ) + action.add_argument( + "--probe-url", + help=( + "Later cell's verified probe listener URL; no flag refreshes the first cell's " + "router-configured addresses" + ), + ) + parser.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") + parser.add_argument( + "--wait", + type=float, + default=600, + help=( + "Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)" + ), + ) + parser.epilog = ( + "Finish/revert require foundation-capable runtimes and completed " + "positive legacy-process turnover. A census cannot verify that external " + "inventory. Address refresh requires every router updated first; retain " + "old endpoints until every router refreshes or restarts after the " + "directory change." + ) + parser.set_defaults(func=cmd_update) diff --git a/server/cli/cell_update_addresses.py b/server/cli/cell_update_addresses.py new file mode 100644 index 0000000000..9d9812ae5e --- /dev/null +++ b/server/cli/cell_update_addresses.py @@ -0,0 +1,64 @@ +"""Refresh proved probe/upstream addresses without claiming fleet-wide cache agreement.""" + +from __future__ import annotations + +import json +from typing import Any + +from server.cli import cell_control, cell_update_probe +from server.router.app import _cell_address +from server.router.directory import CONTROL_FIELDS + + +def normalise(value: str) -> str: + address = _cell_address(value) + if not address: + raise cell_control._refused("probe address is an HTTP(S) host-only URL") + return address + + +async def refresh(run: Any) -> dict[str, Any]: + if run.row["is_first"]: + if run.args.probe_url: + raise cell_control._refused( + "the first cell refreshes router-configured addresses without --probe-url" + ) + proposal = await run.call("/addresses") + if json.dumps(cell_control.preconditions(proposal), sort_keys=True) != json.dumps( + cell_control.preconditions(run.row), sort_keys=True + ): + raise cell_control._refused( + "directory ownership changed while reading router addresses" + ) + probe_url = normalise(proposal.get("router_upstream_probe_url") or "") + upstream_url = normalise(proposal.get("router_upstream_url") or "") + else: + if not run.args.probe_url: + raise cell_control._refused("a later cell probe refresh requires --probe-url") + probe_url, upstream_url = normalise(run.args.probe_url), run.row["upstream_url"] + answer = await cell_update_probe.post( + run, "/addresses", {"probe_url": probe_url, "upstream_url": upstream_url} + ) + if ( + answer["mutation_generation"] != run.row["mutation_generation"] + or any( + json.dumps(answer[name], sort_keys=True) != json.dumps(run.row[name], sort_keys=True) + for name in CONTROL_FIELDS + if name not in ("probe_url", "upstream_url") + ) + or (answer["probe_url"], answer["upstream_url"]) != (probe_url, upstream_url) + ): + raise cell_control._refused( + "address acknowledgement differs from the proved addresses; outcome unknown" + ) + return { + "cell": run.args.cell, + "probe_url": probe_url, + "upstream_url": upstream_url, + "router_refresh_required": bool(run.row["is_first"]), + "next_step": ( + "Keep old endpoints reachable until every router refreshes its routing " + "table or restarts after this directory update; this command cannot " + "prove all-router agreement" + ), + } diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py new file mode 100644 index 0000000000..72a5cb0b37 --- /dev/null +++ b/server/cli/cell_update_participants.py @@ -0,0 +1,256 @@ +"""A post-cutoff participant census and permanent retirement barrier for update.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +from typing import Any, cast +from uuid import UUID + +from server.cli import cell_control, cell_scan, cell_writers +from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology +from server.core import cell, ch_names, instances +from server.core.ch import query_tag +from server.core.provider_process import ( + certified, + has_certificate, + participant_tag, + proves_stopped, +) +from server.insights.store import provider_participants as participants + + +class CompleteReads: + """Pass complete uncached read settings to the existing instance reader.""" + + def __init__(self, ch: Any) -> None: + self.ch = ch + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return cast( + list[dict[str, Any]], + await self.ch.query_all(complete(sql, sequential=1), params or {}), + ) + + +def complete(sql: str, *, sequential: int = 0) -> str: + return ( + sql + + " " + + WHOLE_SQL + + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " + + f"select_sequential_consistency={sequential}" + ) + + +def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for row in rows: + try: + identity = str(UUID(str(row["participant_id"]))) + version = row["version"] + state, proof = row["state"], row["host_proof"] + except (KeyError, TypeError, ValueError): + raise cell_control._refused("participant census contains an unreadable row") from None + if ( + type(version) is not int + or version <= 0 + or state not in ("running", "drained", "retired") + or not isinstance(proof, str) + ): + raise cell_control._refused("participant census contains an unreadable row") + previous = out.get(identity) + if ( + previous is not None + and int(previous["version"]) == version + and (previous["state"], previous["host_proof"]) != (state, proof) + ): + raise cell_control._refused(f"participant {identity} has conflicting equal versions") + if previous is None or int(previous["version"]) < version: + out[identity] = dict(row) + return out + + +def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return { + identity: row + for identity, row in latest(rows).items() + if row["state"] != "retired" or not has_certificate(row["host_proof"]) + } + + +async def census(ch: Any) -> dict[str, dict[str, Any]]: + """Synchronize after the durable cutoff; every configured replica must answer.""" + topology = await read_topology(ch) + if topology.cluster and not topology.entries: + raise cell_control._refused("participant census cannot resolve the configured cluster") + if problems := topology.refusals(): + raise Refused(problems) + names = await cell_writers.host_names(ch, topology) if topology.multi else {} + if problems := (await cell_writers.own_refusals(ch, topology, names))[0]: + raise Refused(problems) + database = ch_names.database() + tables = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " + "name='_provider_participants'" + ), + {"db": database}, + ) + if not tables or any(len(rows) != 1 for rows in tables.values()): + raise cell_control._refused("participant table is missing from a configured replica") + engines = {str(rows[0]["engine"]) for rows in tables.values()} + if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): + raise cell_control._refused( + "participant replicas have unreadable or different table engines" + ) + engine = next(iter(engines)) + if engine.startswith("Replicated"): + rows = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " + "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " + "table='_provider_participants'" + ), + {"db": database}, + ) + if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas( + row for value in rows.values() for row in value + ): + raise cell_control._refused("participant replica set includes an unverified replica") + budget = cell_scan.Budget(cell_control.TRANSACTION_S) + await cell_scan._system( + ch, + f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} " + f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", + topology, + budget, + "participant census", + ) + source = f"{quote(database)}.`_provider_participants`" + params = {} + if topology.multi: + source = f"clusterAllReplicas({{cluster:String}}, {source})" + params["cluster"] = topology.cluster + sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" + rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) + return latest(rows) + + +async def stopped(ch: Any, initial: dict[str, Any], current: dict[str, Any] | None) -> bool: + if current is not None: + try: + same_identity = certified(initial["host_proof"]) == current["host_proof"] + except ValueError: + same_identity = False + if ( + current["state"] == "retired" + and has_certificate(current["host_proof"]) + and same_identity + and int(current["version"]) > int(initial["version"]) + ): + return True + if current is not None: + try: + if certified(initial["host_proof"]) != certified(current["host_proof"]): + return False + except ValueError: + if initial["host_proof"] != current["host_proof"]: + return False + if not proves_stopped(initial["host_proof"]): + return False + # Never use the model protocol's missing-cluster local fallback as death proof. + row = await ch.query_one( + complete(participants.running_sql()), + {"tag": participant_tag(initial["participant_id"])}, + ) + return row is not None and type(row.get("running")) is int and row["running"] == 0 + + +async def replacement( + ch: Any, run: Any, rows: dict[str, dict[str, Any]], captured: set[str], target: dict[str, Any] +) -> bool: + reader = CompleteReads(ch) + live = await instances.read_live(reader, all_deployments=True) + reported = await reader.query_all( + f"SELECT instance_id, cell, tenancy_mode, fingerprint FROM {instances.CELL_TABLE} " + "WHERE has({ids:Array(String)}, instance_id)", + {"ids": [str(row["instance_id"]) for row in live]}, + ) + by_instance = {str(row["instance_id"]): row for row in reported} + for instance in live: + beat = by_instance.get(str(instance["instance_id"])) + if not beat or not instance["serving"] or beat["cell"] != run.args.cell: + continue + try: + value = json.loads(beat["fingerprint"]) + identity = str(UUID(value["participant_id"])) + except (ValueError, TypeError, KeyError): + continue + participant = rows.get(identity) + if ( + identity in captured + or participant is None + or participant["state"] == "retired" + or not has_certificate(participant["host_proof"]) + ): + continue + if ( + value.get("serves") is not True + or value.get("marked") is not True + or (run.row["is_first"] and value.get("router") is not True) + ): + continue + if beat["tenancy_mode"] not in ("recording", "enforced") or not isinstance( + value.get("fields"), dict + ): + continue + if cell.matches(value["fields"], target): + return True + return False + + +async def settle(run: Any, target: dict[str, Any]) -> None: + """Fix one complete captured set; expiry never removes an obligation.""" + with query_tag("cell.update.participants"): + initial = await run.owner.call(lambda: census(run.ch), total_s=cell_control.TRANSACTION_S) + captured = capture(list(initial.values())) + done: set[str] = set() + deadline = time.monotonic() + run.args.wait + while True: + await cell_control.check_lock(run.owner.conn) + + async def poll() -> bool: + rows = await census(run.ch) + for identity, original in captured.items(): + if identity not in done and await stopped(run.ch, original, rows.get(identity)): + done.add(identity) + return await replacement(run.ch, run, rows, set(captured), target) + + live = await run.owner.call(poll, total_s=cell_control.TRANSACTION_S) + remaining = sorted(set(captured) - done) + if not remaining and live: + return + if time.monotonic() >= deadline: + blocked = ", ".join(remaining) or "no captured participant remains" + raise cell_control._refused( + f"participant turnover incomplete ({blocked}); " + f"serving target replacement={live}. " + "Directory stays dual and local acceptance stays single. Online finalization " + "requires foundation code and positive legacy-process turnover; this census " + "cannot verify that external inventory" + ) + print( + f"loom cell update: waiting for participants {', '.join(remaining) or 'none'}; " + f"serving target replacement={live}", + file=sys.stderr, + ) + await asyncio.sleep(min(1, max(0, deadline - time.monotonic()))) diff --git a/server/cli/cell_update_pg.py b/server/cli/cell_update_pg.py new file mode 100644 index 0000000000..ce7cdebcde --- /dev/null +++ b/server/cli/cell_update_pg.py @@ -0,0 +1,128 @@ +"""Update snapshots and acceptance writes on the exact ATTACH-owning backend.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from server.cli import cell_control +from server.core import cell +from server.router import fingerprints + + +@dataclass(frozen=True) +class State: + deployment_id: str + is_first: bool + cell: str + mode: str + current: dict[str, Any] + pending: dict[str, Any] | None + has_tenancy: bool + + def identity(self) -> tuple[Any, ...]: + return ( + self.deployment_id, + self.is_first, + self.cell, + self.mode, + json.dumps((self.current, self.pending), sort_keys=True), + ) + + +async def read(conn: Any, name: str) -> State: + row = await conn.fetchrow( + "SELECT a.deployment_id, f.is_first, f.cell, m.mode, s.current_fp, s.next_fp, " + "EXISTS(SELECT 1 FROM loom.tenancy) AS has_tenancy FROM loom.authority a " + "LEFT JOIN loom.cell_first f ON true LEFT JOIN loom.tenancy_mode m ON true " + "LEFT JOIN loom.cell_self s ON true" + ) + if ( + row is None + or not row["deployment_id"] + or type(row["is_first"]) is not bool + or row["cell"] != name + or row["mode"] not in ("recording", "enforced") + ): + raise cell_control._refused( + "update needs complete prepared state naming this cell; prepare or restore it first" + ) + current, pending = cell.json_object(row["current_fp"]), cell.json_object(row["next_fp"]) + if not fingerprints.readable(current) or ( + pending is not None and not fingerprints.readable(pending) + ): + raise cell_control._refused("update needs readable local current/next fingerprints") + assert current is not None + return State( + str(row["deployment_id"]), + row["is_first"], + name, + row["mode"], + current, + pending, + bool(row["has_tenancy"]), + ) + + +def paired(state: State, registered: dict[str, Any]) -> None: + if ( + state.deployment_id != registered["deployment_id"] + or state.is_first != registered["is_first"] + ): + raise cell_control._refused( + "the prepared local identity differs from the directory registration" + ) + if registered["current_fp"] is not None: + accepted = { + json.dumps(value, sort_keys=True) + for value in (registered["current_fp"], registered["next_fp"]) + if value is not None + } + if any( + json.dumps(value, sort_keys=True) not in accepted + for value in (state.current, state.pending) + if value is not None + ): + raise cell_control._refused( + "local fingerprints differ from the directory reservation; restore the " + "recorded pair" + ) + + +async def write( + owner: cell_control.Owner, + expected: State, + registered: dict[str, Any], + current: dict[str, Any], + pending: dict[str, Any] | None, + placed_orgs: list[str], +) -> State: + cell_control.preconditions(registered) + + async def change() -> State: + conn = owner.conn + await cell_control.paired(conn, registered) + actual = await read(conn, expected.cell) + if actual.identity() != expected.identity(): + raise cell_control._refused( + "local acceptance changed; stop and rerun with fresh observations" + ) + if placed_orgs: + await conn.execute( + "INSERT INTO loom.tenancy(org) SELECT org FROM unnest($1::text[]) AS org " + "ON CONFLICT DO NOTHING", + placed_orgs, + ) + result = await conn.execute( + "UPDATE loom.cell_self SET current_fp=$1, next_fp=$2, updated_at=now() WHERE id=1", + current, + pending, + ) + if result != "UPDATE 1": + raise cell_control._refused( + "local acceptance row disappeared; restore it before updating" + ) + return await read(conn, expected.cell) + + return await owner.transaction(change) diff --git a/server/cli/cell_update_probe.py b/server/cli/cell_update_probe.py new file mode 100644 index 0000000000..7d3dbe2902 --- /dev/null +++ b/server/cli/cell_update_probe.py @@ -0,0 +1,44 @@ +"""Fresh store proof using the existing bounded probe retries and exact nonce cleanup.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from server.cli import cell_control, cell_prepare +from server.router.app import REGISTER_ANSWER_MAX_S + +# Read/probe/commit bounds already used by prepare, plus its reply margin. +ANSWER_S = REGISTER_ANSWER_MAX_S + 10.0 + + +async def post(run: Any, path: str, body: dict[str, Any]) -> dict[str, Any]: + await cell_control.check_lock(run.owner.conn) + nonces = cell_prepare._new_nonces() + + async def clean() -> None: + await cell_prepare._delete_nonces(run.ch, nonces) + + try: + await run.owner.call( + lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=cell_control.TRANSACTION_S + ) + async with asyncio.timeout(run.args.wait + ANSWER_S): + answer = await cell_prepare._register( + run.args, + run.router, + {**body, "nonces": nonces, **cell_control.preconditions(run.row)}, + run.transport, + path=path, + command="update", + ) + await cell_control.check_lock(run.owner.conn) + cell_control.preconditions(answer) + except BaseException: + try: + await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + except BaseException: # noqa: BLE001 — preserve the original operation failure. + pass + raise + await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + return answer diff --git a/server/cli/cell_update_run.py b/server/cli/cell_update_run.py new file mode 100644 index 0000000000..b011f59eff --- /dev/null +++ b/server/cli/cell_update_run.py @@ -0,0 +1,229 @@ +"""Directory-first reservation and local-first narrowing under one retained owner.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import httpx + +from server.cli import cell_control, cell_update_config +from server.cli import cell_update_pg as local +from server.cli import cell_update_probe as probe +from server.core import cell, deployment_config, pg_profile +from server.router import fingerprints +from server.router.directory import CONTROL_FIELDS +from server.settings import settings + + +def same(a: Any, b: Any) -> bool: + return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + + +class Run: + def __init__( + self, ch: Any, args: argparse.Namespace, router: httpx.URL, transport: Any + ) -> None: + self.ch, self.args, self.router, self.transport = ch, args, router, transport + self.owner: cell_control.Owner + self.row: dict[str, Any] = {} + self.state: local.State + + async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]: + return await cell_control.request( + self.router, + self.args.cell, + self.transport, + path, + body, + method="GET" if body is None else "POST", + ) + + async def durable(self) -> local.State: + async def read() -> local.State: + if await pg_profile.standby_now() != "synchronous": + raise cell_control._refused("update requires an attached synchronous standby") + found = await local.read(self.owner.conn, self.args.cell) + if await pg_profile.standby_now() != "synchronous": + raise cell_control._refused("the synchronous standby disappeared during update") + return found + + return await self.owner.transaction(read) + + async def start(self, owner: cell_control.Owner) -> dict[str, Any]: + self.owner = owner + self.state = await self.durable() + self.row = await self.call("") + local.paired(self.state, self.row) + self.target: dict[str, Any] | None = None + if self.args.config: + self.target = cell_update_config.target_fingerprint( + cell_update_config.read_target(Path(self.args.config), settings), + self.state.deployment_id, + self.row, + ) + if self.args.config or self.args.finish or self.args.revert: + self.selection() + elif self.row["is_first"] and self.args.probe_url: + raise cell_control._refused( + "the first cell refreshes router-configured addresses without --probe-url" + ) + elif not self.row["is_first"] and not self.args.probe_url: + raise cell_control._refused("a later cell probe refresh requires --probe-url") + self.row = await cell_control.claim(owner.conn, self.call, self.row) + local.paired(self.state, self.row) + if self.args.config or self.args.finish or self.args.revert: + return await self.configuration() + from server.cli.cell_update_addresses import refresh + + return await refresh(self) + + def check_answer( + self, answer: dict[str, Any], current: dict[str, Any], pending: dict[str, Any] | None + ) -> None: + cell_control.preconditions(answer) + if ( + answer["mutation_generation"] != self.row["mutation_generation"] + or any( + not same(answer[name], self.row[name]) + for name in CONTROL_FIELDS + if name not in ("current_fp", "next_fp") + ) + or not same((answer["current_fp"], answer["next_fp"]), (current, pending)) + ): + raise cell_control._refused( + "directory update acknowledgement differs from its intended result; outcome unknown" + ) + + async def held_orgs(self, current: dict[str, Any], pending: dict[str, Any] | None) -> list[str]: + orgs = sorted( + { + org + for value in (current, pending) + if value is not None + for org in value["fields"].get("configured_orgs", []) + } + ) + result = await cell_control.mutate( + self.owner.conn, + self.call, + self.row, + "/configuration/placements", + {"orgs": orgs, "keys_fp": deployment_config.fingerprints(settings)["keys_fp"]}, + ) + held = result.get("orgs") + if ( + not isinstance(held, list) + or any(type(org) is not str for org in held) + or len(set(held)) != len(held) + or not set(held) <= set(orgs) + ): + raise cell_control._refused( + "directory returned an unreadable configured-placement proof" + ) + if set(held) != set(orgs) and not ( + self.row["is_first"] and self.row["state"] == "attaching" + ): + raise cell_control._refused( + "configured placement is missing; restore directory metadata before updating" + ) + return held + + async def publish(self, target: dict[str, Any], action: str) -> dict[str, Any]: + keys = deployment_config.fingerprints(settings)["keys_fp"] + return await probe.post( + self, "/configuration", {"action": action, "target_fp": target, "keys_fp": keys} + ) + + def selection( + self, + ) -> tuple[ + str, + bool, + dict[str, Any], + dict[str, Any] | None, + dict[str, Any], + dict[str, Any], + dict[str, Any] | None, + ]: + action = "config" if self.args.config else "finish" if self.args.finish else "revert" + recorded = self.row["current_fp"] is not None + current = self.row["current_fp"] if recorded else self.state.current + pending = self.row["next_fp"] if recorded else self.state.pending + target = ( + self.target + if action == "config" + else pending + if action == "finish" and pending is not None + else current + ) + assert target is not None + desired, nxt = fingerprints.configuration_pair(current, pending, target, action) + if reason := fingerprints.registration_pair_refusal(self.row, desired, nxt): + raise cell_control._refused(reason) + return action, recorded, current, pending, target, desired, nxt + + async def configuration(self) -> dict[str, Any]: + action, recorded, current, pending, target, desired, nxt = self.selection() + old_pending = pending is not None + if action == "config" and recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, nxt) + self.row = answer + held = await self.held_orgs(self.row["current_fp"], self.row["next_fp"]) if recorded else [] + self.state = await local.write(self.owner, self.state, self.row, desired, nxt, held) + self.state = await self.durable() + if not same((self.state.current, self.state.pending), (desired, nxt)): + raise cell_control._refused("local acceptance differs after its durable update") + if action != "config" and old_pending: + from server.cli.cell_update_participants import settle + + await settle(self, target) + self.state = await self.durable() + if not same((self.state.current, self.state.pending), (target, None)): + raise cell_control._refused("local cutoff changed while participants turned over") + # A fresh full-row read refuses a delayed claim before final reservation release. + latest = await self.call("") + if not same(cell_control.preconditions(latest), cell_control.preconditions(self.row)): + raise cell_control._refused( + "directory ownership or reservation changed; rerun with a fresh census" + ) + if recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, None) + self.row = answer + elif action != "config" and recorded: + answer = await self.publish(target, action) + self.check_answer(answer, desired, None) + self.row = answer + return { + "cell": self.args.cell, + "action": action, + "directory_reserved": recorded, + "current": cell.digest(desired["fields"]), + "next": None if nxt is None else cell.digest(nxt["fields"]), + "changed_fields": sorted( + name + for name in set(current["fields"]) | set(target["fields"]) + if not same( + current["fields"].get(name, cell.DEFAULTS.get(name)), + target["fields"].get(name, cell.DEFAULTS.get(name)), + ) + ), + "rollout": ( + "reserved; roll out the target configuration before finish" + if recorded + else "local pair prepared; attach must reserve both configurations" + ) + if action == "config" and nxt is not None + else "complete", + } + + +async def update( + ch: Any, args: argparse.Namespace, router: httpx.URL, transport: Any = None +) -> dict[str, Any]: + run = Run(ch, args, router, transport) + return await cell_control.run("update", run.start) diff --git a/server/router/app.py b/server/router/app.py index cb408f56e9..87cc8d775e 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -53,6 +53,7 @@ from collections import OrderedDict from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager +from dataclasses import replace from typing import Any, NamedTuple, Protocol from urllib.parse import parse_qs from uuid import UUID @@ -725,6 +726,101 @@ async def claim(self, request: Request) -> JSONResponse: lambda: directory.claim(request.path_params["cell"], expected), _attach_answer ) + async def addresses(self, request: Request) -> JSONResponse: + if request.method == "GET": + if refused := self._operator(request) or _malformed_cell_refusal(request): + return refused + found = await _read_attaching(request.path_params["cell"]) + if isinstance(found, JSONResponse): + return found + return JSONResponse( + { + **_attach_answer(found), + "router_upstream_url": _cell_address(settings.router_upstream_url) or "", + "router_upstream_probe_url": _cell_address(settings.router_upstream_probe_url) + or "", + } + ) + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + if set(body) != {"probe_url", "upstream_url", "nonces"} or not _nonces_shaped( + body.get("nonces") + ): + return _refused( + 400, "address refresh requires probe_url, upstream_url and store nonces" + ) + probe_text = _text(body, "probe_url", needed=True) + upstream_text = _text(body, "upstream_url") + probe_url = _cell_address(probe_text) if probe_text is not None else None + upstream_url = _cell_address(upstream_text) if upstream_text else "" + if not probe_url or upstream_text is None or upstream_url is None: + return _refused(400, "address refresh requires normalized HTTP(S) host-only addresses") + found = await _read_attaching(request.path_params["cell"]) + if isinstance(found, JSONResponse): + return found + if reason := directory.control_refusal(found, expected): + return _refused(409, reason) + proposed = replace(found, probe_url=probe_url, upstream_url=upstream_url) + + def configured() -> bool: + return not found.row.is_first or (probe_url, upstream_url) == ( + _cell_address(settings.router_upstream_probe_url), + _cell_address(settings.router_upstream_url), + ) + + if not configured(): + return _refused( + 409, "proposed addresses differ from this router's configured upstreams" + ) + if not found.row.is_first and upstream_url != found.upstream_url: + return _refused(409, "a later cell probe refresh cannot change its upstream address") + if refused := await _probe( + proposed, body["nonces"], self._probe_transport, writer="update" + ): + return refused + if found.row.is_first: + if refused := await _upstream_check(proposed, self._probe_transport): + return refused + response = await _answered( + lambda: directory.update_addresses( + found.row.cell, probe_url, upstream_url, expected=expected, configured=configured + ), + _attach_answer, + ) + if response.status_code in (200, 503): + self._table_at = None + return response + + async def configuration_placements(self, request: Request) -> JSONResponse: + body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) + if isinstance(body, JSONResponse): + return body + expected = _control(body) + if isinstance(expected, JSONResponse): + return expected + orgs = body.get("orgs") + if ( + set(body) != {"orgs", "keys_fp"} + or _text(body, "keys_fp", needed=True) is None + or not isinstance(orgs, list) + or any(cell_state.named_org(org) is None for org in orgs) + ): + return _refused( + 400, "placement proof requires keys_fp and an explicit list of configured org names" + ) + if any(cell_state.org_name_refusal(org) for org in orgs): + return _refused(400, "a configured org name is too long", code="ORG_TOO_LONG") + return await _answered( + lambda: directory.configuration_placements( + request.path_params["cell"], orgs, expected=expected, keys_fp=body["keys_fp"] + ), + lambda held: {"cell": request.path_params["cell"], "orgs": held}, + ) + async def configuration(self, request: Request) -> JSONResponse: body = await self._operator_body(request, ATTACH_BEGIN_MAX_BYTES) if isinstance(body, JSONResponse): @@ -737,8 +833,8 @@ async def configuration(self, request: Request) -> JSONResponse: action, target, keys_fp = body["action"], body["target_fp"], body["keys_fp"] if action not in ("config", "finish", "revert") or not fingerprints.readable(target): return _refused(400, "configuration requires a supported action and readable target_fp") - if not isinstance(keys_fp, str) or not cell_state.SHA256.fullmatch(keys_fp): - return _refused(400, "keys_fp is a SHA-256 fingerprint") + if _text(body, "keys_fp", needed=True) is None: + return _refused(400, f"keys_fp is text of 1 to {_TEXT_MAX} characters, no NUL") if not _nonces_shaped(body["nonces"]): return _refused(400, _NONCES_REFUSAL) name = request.path_params["cell"] @@ -1544,6 +1640,16 @@ async def _lifespan(_app: Starlette) -> AsyncIterator[None]: ), Route("/cells/v1/operator/cells/{cell}", router.operator_cell, methods=["GET"]), Route("/cells/v1/operator/cells/{cell}/claim", router.claim, methods=["POST"]), + Route( + "/cells/v1/operator/cells/{cell}/addresses", + router.addresses, + methods=["GET", "POST"], + ), + Route( + "/cells/v1/operator/cells/{cell}/configuration/placements", + router.configuration_placements, + methods=["POST"], + ), Route( "/cells/v1/operator/cells/{cell}/configuration", router.configuration, diff --git a/server/router/directory.py b/server/router/directory.py index e2621feac9..2302f31c4b 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1044,6 +1044,86 @@ async def change_locked(conn: Any, _found: Attaching) -> Attaching: return await _attach_change(cell_name, expected, change_locked, claim=True) +async def update_addresses( + cell_name: str, + probe_url: str, + upstream_url: str, + *, + expected: Control, + configured: Callable[[], bool], +) -> Attaching: + """Commit only the addresses whose stores were just proved outside this transaction.""" + + async def change(conn: Any, found: Attaching) -> Attaching: + if not configured(): + raise OperatorRefused( + "the contacted router's configured addresses changed during proof" + ) + if not found.row.is_first and upstream_url != found.upstream_url: + raise OperatorRefused("a later cell probe refresh cannot change its upstream address") + await conn.execute( + ( + "UPDATE loom_router.cells SET probe_url=$2, upstream_url=$3, updated_at=now() " + "WHERE cell=$1" + ), + cell_name, + probe_url, + upstream_url, + ) + await _audit( + conn, + "addresses", + {"cell": cell_name, "probe_url": probe_url, "upstream_url": upstream_url}, + ) + row = await conn.fetchrow(f"{_CELL_ATTACH} WHERE cell=$1", cell_name) + return _attaching(row, found.other) + + return await _attach_change(cell_name, expected, change) + + +async def configuration_placements( + cell_name: str, orgs: list[str], *, expected: Control, keys_fp: str +) -> list[str]: + """Acknowledge actual immutable winners before the cell repairs local tenancy.""" + _refuse_long(orgs) + + async def read(conn: Any, found: Attaching) -> list[str]: + if ( + await conn.fetchval("SELECT keys_fp FROM loom_router.cells WHERE cell=$1", cell_name) + != keys_fp + ): + raise OperatorRefused( + "cell root-secret fingerprint changed; configuration update refuses" + ) + accepted = [ + value for value in (found.row.current_fp, found.row.next_fp) if value is not None + ] + if not accepted or any(not fingerprints.readable(value) for value in accepted): + raise OperatorRefused( + "configured placement proof needs readable directory reservations" + ) + configured = { + org for value in accepted for org in value["fields"].get("configured_orgs", []) + } + if not set(orgs) <= configured: + raise OperatorRefused("placement proof names an org outside the reserved configuration") + rows = await conn.fetch( + "SELECT org, cell FROM loom_router.placements WHERE org=ANY($1::text[]) ORDER BY org", + sorted(set(orgs)), + ) + for row in rows: + if row["cell"] != cell_name: + raise OperatorRefused(f"org {row['org']!r} is placed in cell {row['cell']!r}") + await _audit( + conn, + "configuration_placements", + {"cell": cell_name, "named": len(set(orgs)), "held": len(rows)}, + ) + return [str(row["org"]) for row in rows] + + return await _attach_change(cell_name, expected, read) + + async def update_configuration( cell_name: str, target: dict[str, Any], diff --git a/tests/integration/test_cell_update_addresses.py b/tests/integration/test_cell_update_addresses.py new file mode 100644 index 0000000000..812c8cfc63 --- /dev/null +++ b/tests/integration/test_cell_update_addresses.py @@ -0,0 +1,172 @@ +"""Address refresh proves listeners before the generation-fenced directory write.""" + +from __future__ import annotations + +import hashlib +from dataclasses import replace + +import httpx +import pytest + +from server.router import app, directory +from server.settings import settings +from tests.integration.test_cell_update_directory import configured as configured +from tests.integration.test_cell_update_directory import observed +from tests.integration.test_router_directory_pg import dsn as dsn + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first", [False, True]) +async def test_address_refresh_changes_only_verified_addresses(configured, monkeypatch, first): + name = "first" if first else "other" + token = "lco_address-proof" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + monkeypatch.setattr(settings, "router_upstream_url", "http://new-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", "http://new-probe") + expected = await observed(name) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + + def network(request): + if request.url.host == "new-main": + return httpx.Response(200, json={"deployment_id": f"dep-{name}"}) + return httpx.Response( + 200, + json={ + "deployment_id": f"dep-{name}", + "bucket": {"name": "", "prefix": ""}, + "nonces": nonces, + }, + ) + + before = await directory.attach_read(name) + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://router" + ) as client: + response = await client.post( + f"/cells/v1/operator/cells/{name}/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": "http://new-probe", + "upstream_url": "http://new-main" if first else before.upstream_url, + "nonces": nonces, + }, + ) + assert response.status_code == 200, response.text + after = await directory.attach_read(name) + assert after == replace( + before, + probe_url="http://new-probe", + upstream_url="http://new-main" if first else before.upstream_url, + ) + + +@pytest.mark.asyncio +async def test_stale_address_generation_cannot_overwrite_a_later_refresh(configured): + expected = await observed("first") + await observed("first") + before = await directory.attach_read("first") + with pytest.raises(directory.OperatorRefused, match="generation"): + await directory.update_addresses( + "first", "http://probe", "http://main", expected=expected, configured=lambda: True + ) + assert await directory.attach_read("first") == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "invalid", ["main-port", "deployment", "bucket", "nonce", "upstream", "config-changed"] +) +async def test_invalid_address_proof_leaves_exact_directory_unchanged( + configured, monkeypatch, invalid +): + token = "lco_address-refusal" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + monkeypatch.setattr(settings, "router_upstream_url", "http://new-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", "http://new-probe") + expected = await observed("first") + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + before = await directory.attach_read("first") + + def network(request): + if request.url.host == "new-main": + return httpx.Response( + 200, json={"deployment_id": "wrong" if invalid == "upstream" else "dep-first"} + ) + answer = { + "deployment_id": "dep-first", + "bucket": {"name": "", "prefix": ""}, + "nonces": nonces, + } + if invalid == "main-port": + answer.pop("nonces") + elif invalid == "deployment": + answer["deployment_id"] = "other" + elif invalid == "bucket": + answer["bucket"] = {"name": "other", "prefix": ""} + elif invalid == "nonce": + answer["nonces"] = nonces | {"postgres": "old"} + elif invalid == "config-changed": + monkeypatch.setattr(settings, "router_upstream_url", "http://changed-main") + return httpx.Response(200, json=answer) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://router" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": "http://new-probe", + "upstream_url": "http://new-main", + "nonces": nonces, + }, + ) + assert response.status_code == 409, response.text + assert await directory.attach_read("first") == before + + +@pytest.mark.asyncio +async def test_old_router_config_can_reverse_addresses_with_fresh_expected_row( + configured, monkeypatch +): + # The all-router rollout prerequisite cannot be inferred from a successful CAS. + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + token = "lco_router-rollout" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + + def network(request): + answer = {"deployment_id": "dep-first"} + if "probe" in request.url.host: + answer |= {"bucket": {"name": "", "prefix": ""}, "nonces": nonces} + return httpx.Response(200, json=answer) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app(router)), base_url="http://router" + ) as client: + for version in ("new", "old"): + monkeypatch.setattr(settings, "router_upstream_url", f"http://{version}-main") + monkeypatch.setattr(settings, "router_upstream_probe_url", f"http://{version}-probe") + expected = await observed("first") + response = await client.post( + "/cells/v1/operator/cells/first/addresses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "expected": expected.values | {"mutation_generation": str(expected.generation)}, + "probe_url": f"http://{version}-probe", + "upstream_url": f"http://{version}-main", + "nonces": nonces, + }, + ) + assert response.status_code == 200, response.text + assert (await directory.attach_read("first")).upstream_url == "http://old-main" diff --git a/tests/integration/test_cell_update_directory.py b/tests/integration/test_cell_update_directory.py index 3a2d95007c..0b7d8aed26 100644 --- a/tests/integration/test_cell_update_directory.py +++ b/tests/integration/test_cell_update_directory.py @@ -367,3 +367,106 @@ async def stores_check(self, found, nonces): ) else: assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_configuration_accepts_the_actual_instance_key_fingerprint(configured, monkeypatch): + from server.core.deployment_config import fingerprints as instance_fingerprints + + keys = instance_fingerprints(settings)["keys_fp"] + assert len(keys) == 32 + await configured.execute("UPDATE loom_router.cells SET keys_fp=$1 WHERE cell='first'", keys) + token = "lco_configuration-real-keys" + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + + async def stores_check(*_): + return None + + monkeypatch.setattr(app.Router, "_stores_check", stores_check) + compare = await observed("first") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app.build_app()), base_url="http://router" + ) as client: + response = await client.post( + "/cells/v1/operator/cells/first/configuration", + headers={"Authorization": f"Bearer {token}"}, + json={ + "action": "config", + "target_fp": fingerprint("first", "new"), + "keys_fp": keys, + "nonces": {"postgres": "pg", "clickhouse": "ch", "bucket": None}, + "expected": compare.values | {"mutation_generation": str(compare.generation)}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["next_fp"] == fingerprint("first", "new") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("winner", ["first", "other", None]) +async def test_configuration_placement_read_proves_actual_winners_without_mutation( + configured, winner +): + target = fingerprint("first", "new", ("configured",)) + await configured.execute( + "UPDATE loom_router.cells SET next_fp=$1::jsonb WHERE cell='first'", json.dumps(target) + ) + if winner is not None: + await configured.execute( + "INSERT INTO loom_router.placements(org,cell) VALUES ('configured',$1)", winner + ) + await configured.execute( + "UPDATE loom_router.cells SET placed=placed+1 WHERE cell=$1", winner + ) + before = await stored(configured) + compare = await observed("first") + if winner == "other": + with pytest.raises(directory.OperatorRefused, match="other"): + await directory.configuration_placements( + "first", ["configured"], expected=compare, keys_fp="c" * 64 + ) + else: + assert await directory.configuration_placements( + "first", ["configured"], expected=compare, keys_fp="c" * 64 + ) == (["configured"] if winner else []) + assert await stored(configured) == before + + +@pytest.mark.asyncio +async def test_placement_proof_refuses_root_secret_drift_before_local_repair(configured): + with pytest.raises(directory.OperatorRefused, match="root-secret"): + await directory.configuration_placements( + "first", [], expected=await observed("first"), keys_fp="changed" + ) + + +@pytest.mark.asyncio +async def test_placement_proof_read_is_a_bounded_durable_noop(configured, monkeypatch): + expected = await observed("first") + original = directory._connection + statements = [] + before = await stored(configured) + wal_before = await configured.fetchval("SELECT pg_current_wal_insert_lsn()") + + @asynccontextmanager + async def counted(): + async with original() as conn: + conn.add_query_logger(lambda record: statements.append(record.query)) + yield conn + + monkeypatch.setattr(directory, "_connection", counted) + assert ( + await directory.configuration_placements("first", [], expected=expected, keys_fp="c" * 64) + == [] + ) + await asyncio.sleep(0) + assert len(statements) == 9, statements + assert directory._bound_s(len(statements)) == 120 + assert directory.CONTROL_MAX_S == 131 + assert directory._bound_s(len(statements)) <= directory.CONTROL_MAX_S + assert sum("pg_logical_emit_message" in sql for sql in statements) == 1 + assert statements[0] == "BEGIN;" and statements[-1] == "COMMIT;" + assert await configured.fetchval("SELECT pg_current_wal_insert_lsn()") > wal_before + assert await stored(configured) == before diff --git a/tests/integration/test_cell_update_participants_ch.py b/tests/integration/test_cell_update_participants_ch.py new file mode 100644 index 0000000000..5fc5176c72 --- /dev/null +++ b/tests/integration/test_cell_update_participants_ch.py @@ -0,0 +1,142 @@ +"""Update reads a scratch participant census and waits for actual stored successors.""" + +from __future__ import annotations + +import asyncio +import json +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest + +from server.cli import cell_control +from server.cli import cell_update_participants as update +from server.core import cell, instances, provider_process +from server.core.ch import CHClient, query_tag +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_attach import FakeConn + + +@pytest.mark.asyncio +async def test_postcutoff_census_waits_for_certified_old_row_and_serving_successor(monkeypatch): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + from server.settings import settings + + database = "update_" + uuid4().hex + client = CHClient() + await client.connect() + task = None + with query_tag("test.cell_update.participants"): + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + try: + await client.execute( + "CREATE TABLE {db:Identifier}._provider_participants AS loom._provider_participants", + {"db": database}, + ) + for table in ("instances", "cell_heartbeats"): + await client.execute( + f"CREATE TABLE {{db:Identifier}}.{table} AS loom.{table} ENGINE=MergeTree ORDER BY instance_id", + {"db": database}, + ) + monkeypatch.setattr(settings, "ch_database", database) + monkeypatch.setattr(settings, "ch_cluster", "") + old, new = uuid4(), uuid4() + proof = provider_process.certified(provider_process.host_proof()) + await client.insert( + "loom._provider_participants", + [{"participant_id": old, "state": "running", "version": 1, "host_proof": proof}], + ) + await client.execute( + "ALTER TABLE loom._provider_participants UPDATE heartbeat_at=now64(3)-toIntervalHour(25) WHERE 1 SETTINGS mutations_sync=2" + ) + initial = await update.census(client) + assert set(update.capture(list(initial.values()))) == {str(old)} + target = cell.as_value({"deployment_id": "target"}) + captured = asyncio.Event() + actual_census = update.census + reads = 0 + + async def census(ch): + nonlocal reads + result = await actual_census(ch) + reads += 1 + if reads == 1: + captured.set() + return result + + async def successor(): + await captured.wait() + await client.insert( + "loom._provider_participants", + [ + { + "participant_id": old, + "state": "retired", + "version": 2, + "host_proof": proof, + }, + { + "participant_id": new, + "state": "running", + "version": 1, + "host_proof": proof, + }, + ], + ) + instance = instances.row() | { + "instance_id": "successor", + "serving": 1, + "stopped": 0, + } + await client.insert("loom.instances", [instance]) + await client.insert( + "loom.cell_heartbeats", + [ + { + "instance_id": "successor", + "cell": "first", + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + target + | { + "participant_id": str(new), + "serves": True, + "marked": True, + "router": True, + } + ), + } + ], + ) + + monkeypatch.setattr(update, "census", census) + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + run = SimpleNamespace( + ch=client, + args=SimpleNamespace(cell="first", wait=5), + row={"is_first": True}, + owner=owner(FakeConn()), + ) + task = asyncio.create_task(successor()) + await update.settle(run, target) + await task + assert reads >= 2 + assert len(await actual_census(client)) == 2 + # A workers-only heartbeat cannot replace the serving target. + await client.execute("TRUNCATE TABLE loom.instances") + await client.insert( + "loom.instances", [instances.row() | {"instance_id": "successor", "serving": 0}] + ) + assert not await update.replacement( + client, run, await actual_census(client), {str(old)}, target + ) + finally: + if task is not None: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await client.execute("DROP DATABASE {db:Identifier} SYNC", {"db": database}) + await client.disconnect() diff --git a/tests/integration/test_cell_update_run_pg.py b/tests/integration/test_cell_update_run_pg.py new file mode 100644 index 0000000000..f1d314a12f --- /dev/null +++ b/tests/integration/test_cell_update_run_pg.py @@ -0,0 +1,384 @@ +"""The update coordinator commits real local and directory state on isolated stores.""" + +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_control, cell_update_participants, cell_update_pg, cell_update_probe +from server.cli.cell_update_run import Run +from server.core import cell, deployment_config +from server.router import app +from server.settings import settings +from tests.integration.test_cell_control_fence import control_stores as control_stores +from tests.integration.test_cell_control_fence import local_control as local_control +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_update_run import value + +REAL_POST = cell_update_probe.post +REAL_STORES_CHECK = app.Router._stores_check + + +@pytest.fixture +async def update_stores(control_stores, local_control, monkeypatch, tmp_path): + client, directory_conn, _ = control_stores + local_conn, local_dsn = local_control + token = "lco_" + "f" * 128 + monkeypatch.setattr(settings, "router_operator_token", token) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(token.encode()).hexdigest() + ) + current = value() + await directory_conn.execute( + "UPDATE loom_router.cells SET state='serving',mode='enforced', current_fp=$1::jsonb, " + "keys_fp=$2, issuers=ARRAY['old']", + json.dumps(current), + deployment_config.fingerprints(settings)["keys_fp"], + ) + await local_conn.execute("INSERT INTO loom.cell_first(is_first,cell) VALUES(true,'first')") + await local_conn.execute("INSERT INTO loom.tenancy_mode(mode) VALUES('enforced')") + await local_conn.execute("INSERT INTO loom.cell_self(current_fp) VALUES($1::jsonb)", current) + await local_conn.execute("SELECT pg_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + path = tmp_path / "target.yaml" + path.write_text("version: 1\nsettings: {}\n") + events = [] + real_write = cell_update_pg.write + + async def write(owned, expected, row, target, pending, orgs): + events.append("local-write") + # A successor is valid only in a fresh owner after the original backend died. + assert owned.conn is local_conn or local_conn.is_closed() + # The observed directory reservation is already durable before local widening. + recorded = await directory_conn.fetchrow("SELECT current_fp,next_fp FROM loom_router.cells") + assert (json.loads(recorded["current_fp"]) if recorded["current_fp"] else None) == row[ + "current_fp" + ] + assert (json.loads(recorded["next_fp"]) if recorded["next_fp"] else None) == row["next_fp"] + return await real_write(owned, expected, row, target, pending, orgs) + + async def post(run, route, body): + events.append("directory-" + body["action"]) + return await run.call( + route, + { + **body, + **cell_control.preconditions(run.row), + "nonces": {"postgres": "pg", "clickhouse": "ch", "bucket": None}, + }, + ) + + async def stores_checked(*_): + return None + + from server.router.app import Router + + monkeypatch.setattr(Router, "_stores_check", stores_checked) + monkeypatch.setattr(cell_update_probe, "post", post) + monkeypatch.setattr(cell_update_pg, "write", write) + + def make(action="config", target=None): + from server.cli import cell_update_config + + monkeypatch.setattr( + cell_update_config, + "target_fingerprint", + lambda *_: target or value("new", orgs=("permanent",)), + ) + args = Namespace( + cell="first", + config=path if action == "config" else None, + finish=action == "finish", + revert=action == "revert", + probe_url=None, + wait=0, + ) + return Run(None, args, httpx.URL("http://router"), client._transport) + + yield make, local_conn, directory_conn, owner(local_conn), events, local_dsn + + +@pytest.mark.asyncio +async def test_coordinator_config_and_retry_commit_both_real_stores_once(update_stores): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + await make().start(owned) + expected = value("new", orgs=("permanent",)) + local = await cell_update_pg.read(local_conn, "first") + assert local.current == value() and local.pending == expected + assert [dict(row) for row in await local_conn.fetch("SELECT org FROM loom.tenancy")] == [ + {"org": "permanent"} + ] + assert await directory_conn.fetchval("SELECT placed FROM loom_router.cells") == 1 + assert await directory_conn.fetchval("SELECT count(*) FROM loom_router.placements") == 1 + assert events == ["directory-config", "local-write", "directory-config", "local-write"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["finish", "revert"]) +async def test_actual_cutoff_precedes_turnover_and_directory_release( + update_stores, monkeypatch, action +): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + # Lost local repair is reproduced by deleting only this scratch tenancy row. + await local_conn.execute("DELETE FROM loom.tenancy WHERE org='permanent'") + target = value("new", orgs=("permanent",)) if action == "finish" else value() + + async def barrier(run, wanted): + assert wanted == target + local = await cell_update_pg.read(local_conn, "first") + assert local.current == target and local.pending is None + assert await local_conn.fetchval( + "SELECT EXISTS(SELECT 1 FROM loom.tenancy WHERE org='permanent')" + ) + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + events.append("turnover") + + monkeypatch.setattr(cell_update_participants, "settle", barrier) + await make(action).start(owned) + stored = await directory_conn.fetchrow( + "SELECT current_fp,next_fp,placed FROM loom_router.cells" + ) + assert json.loads(stored["current_fp"]) == target and stored["next_fp"] is None + assert stored["placed"] == 1 + assert events[-3:] == ["local-write", "turnover", "directory-" + action] + + +@pytest.mark.asyncio +async def test_directory_commit_then_local_failure_is_repaired_without_duplicate_placement( + update_stores, monkeypatch +): + make, local_conn, directory_conn, owned, _, _ = update_stores + real = cell_update_pg.write + monkeypatch.setattr( + cell_update_pg, "write", AsyncMock(side_effect=OSError("local unavailable")) + ) + with pytest.raises(OSError, match="local unavailable"): + await make().start(owned) + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + assert (await cell_update_pg.read(local_conn, "first")).pending is None + monkeypatch.setattr(cell_update_pg, "write", real) + await make().start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending == value( + "new", orgs=("permanent",) + ) + assert await directory_conn.fetchval("SELECT placed FROM loom_router.cells") == 1 + + +@pytest.mark.asyncio +async def test_turnover_timeout_leaves_real_directory_dual_and_local_single_for_retry( + update_stores, monkeypatch +): + make, local_conn, directory_conn, owned, _, _ = update_stores + await make().start(owned) + monkeypatch.setattr( + cell_update_participants, "settle", AsyncMock(side_effect=TimeoutError("old process")) + ) + with pytest.raises(TimeoutError, match="old process"): + await make("finish").start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending is None + assert await directory_conn.fetchval("SELECT next_fp IS NOT NULL FROM loom_router.cells") + monkeypatch.setattr(cell_update_participants, "settle", AsyncMock()) + await make("finish").start(owned) + assert await directory_conn.fetchval("SELECT next_fp IS NULL FROM loom_router.cells") + + +@pytest.mark.asyncio +async def test_prebegin_real_local_pair_only_never_reserves_or_places(update_stores): + make, local_conn, directory_conn, owned, events, _ = update_stores + await directory_conn.execute( + "UPDATE loom_router.cells SET state='attaching',current_fp=NULL,next_fp=NULL,issuers='{}'" + ) + await make().start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending == value( + "new", orgs=("permanent",) + ) + assert await directory_conn.fetchval( + "SELECT current_fp IS NULL AND next_fp IS NULL FROM loom_router.cells" + ) + assert await directory_conn.fetchval("SELECT count(*) FROM loom_router.placements") == 0 + assert await local_conn.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert events == ["local-write"] + + +@pytest.mark.asyncio +async def test_changed_root_keys_refuse_before_real_local_cutoff(update_stores, monkeypatch): + make, local_conn, directory_conn, owned, events, _ = update_stores + await make().start(owned) + before = await cell_update_pg.read(local_conn, "first") + await directory_conn.execute("UPDATE loom_router.cells SET keys_fp='changed'") + barrier = AsyncMock(side_effect=AssertionError("must not reach barrier")) + monkeypatch.setattr(cell_update_participants, "settle", barrier) + with pytest.raises(cell_control.Refused, match="root-secret fingerprint changed"): + await make("finish").start(owned) + assert await cell_update_pg.read(local_conn, "first") == before + assert events.count("local-write") == 1 + barrier.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("aba", [False, True]) +async def test_delayed_actual_finish_cannot_release_after_backend_death_and_fresh_rewiden( + update_stores, monkeypatch, aba +): + import asyncio + + import asyncpg + + from server.router import directory + + make, local_conn, directory_conn, owned, _, dsn = update_stores + await make().start(owned) + monkeypatch.setattr(cell_update_participants, "settle", AsyncMock()) + entered, resume = asyncio.Event(), asyncio.Event() + real_change = directory.update_configuration + delayed_once = False + + async def delayed(*args, **kwargs): + nonlocal delayed_once + if args[2] == "finish" and not delayed_once: + delayed_once = True + entered.set() + await resume.wait() + return await real_change(*args, **kwargs) + + monkeypatch.setattr(directory, "update_configuration", delayed) + pending = asyncio.create_task(make("finish").start(owned)) + admin = await asyncpg.connect(dsn) + successor = await asyncpg.connect(dsn) + await directory._init(successor) + try: + await asyncio.wait_for(entered.wait(), 5) + old_generation = str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + await admin.execute("SELECT pg_terminate_backend($1)", local_conn.get_server_pid()) + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + fresh_owner = owner(successor) + await make().start(fresh_owner) + if aba: + await make("revert").start(fresh_owner) + await make().start(fresh_owner) + assert old_generation != str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + assert (await cell_update_pg.read(successor, "first")).pending is not None + resume.set() + with pytest.raises(cell_control.Refused, match="generation|attach lock|lock session"): + await pending + stored = await directory_conn.fetchrow( + "SELECT current_fp,next_fp,placed FROM loom_router.cells" + ) + assert json.loads(stored["current_fp"]) == value() + assert json.loads(stored["next_fp"]) == value("new", orgs=("permanent",)) + assert stored["placed"] == 1 + assert (await cell_update_pg.read(successor, "first")).pending is not None + finally: + resume.set() + if not pending.done(): + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await successor.close() + await admin.close() + + +@pytest.mark.asyncio +async def test_local_cas_refuses_changed_state_before_tenancy_repair(update_stores): + make, local_conn, _, owned, _, _ = update_stores + await make().start(owned) + initial = await cell_update_pg.read(local_conn, "first") + run = make("finish") + run.row = await run.call("") + await local_conn.execute("UPDATE loom.cell_self SET next_fp=NULL") + with pytest.raises(cell_control.Refused, match="acceptance changed"): + await cell_update_pg.write(owned, initial, run.row, initial.current, None, ["not-held"]) + # The owner discards the failed transaction; the command cannot reuse the backend. + assert owned.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proof", ["valid", "wrong", "busy"]) +async def test_update_uses_fresh_real_pg_ch_nonces_and_cleans_only_its_values( + update_stores, monkeypatch, proof +): + import os + from uuid import uuid4 + + from server.cli import cell_prepare + from server.core import cell_probe, leases + from server.core.ch import CHClient, query_tag + + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + make, local_conn, directory_conn, owned, _, _ = update_stores + database = "update_nonce_" + uuid4().hex + table = database + ".locks" + client = CHClient() + await client.connect() + with query_tag("test.cell_update.nonces"): + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + try: + await client.execute( + "CREATE TABLE {db:Identifier}.locks (name String, holder String, " + "acquired_at DateTime64(3), renewed_at DateTime64(3), ttl_s UInt32, " + "build_commit String) ENGINE=KeeperMap({path:String}) PRIMARY KEY name", + {"db": database, "path": "/loom/test/" + database}, + ) + monkeypatch.setattr( + leases, "LOCK_RELEASE", leases.LOCK_RELEASE.replace(leases.LOCK_TABLE, table) + ) + monkeypatch.setattr(leases, "LOCK_TABLE", table) + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(cell_update_probe, "post", REAL_POST) + monkeypatch.setattr(app.Router, "_stores_check", REAL_STORES_CHECK) + monkeypatch.setattr(cell_prepare, "_sleep", AsyncMock()) + await directory_conn.execute( + "UPDATE loom_router.cells SET probe_url='http://probe',upstream_url='http://main'" + ) + requests = [] + + async def network(request): + if request.url.host == "main": + return httpx.Response(200, json={"deployment_id": "dep-first"}) + requests.append(request) + actual = await cell_probe.nonces() + assert actual["postgres"] and actual["clickhouse"] + if proof == "busy" and len(requests) == 1: + return httpx.Response(429, headers={"Retry-After": "1"}) + return httpx.Response( + 200, + json={ + "deployment_id": "dep-first", + "bucket": {"name": "", "prefix": ""}, + "nonces": actual | ({"postgres": "old"} if proof == "wrong" else {}), + }, + ) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + run = make() + run.transport = httpx.ASGITransport(app=app.build_app(router)) + run.ch, run.args.wait = client, 3 + if proof == "wrong": + with pytest.raises(cell_prepare.Refused, match="does not read the stores"): + await run.start(owned) + assert await directory_conn.fetchval( + "SELECT next_fp IS NULL FROM loom_router.cells" + ) + assert (await cell_update_pg.read(local_conn, "first")).pending is None + else: + await run.start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending is not None + assert len(requests) == (2 if proof == "busy" else 1) + assert await cell_probe.nonces() == { + "postgres": None, + "clickhouse": None, + "bucket": None, + } + finally: + await client.execute("DROP DATABASE {db:Identifier} SYNC", {"db": database}) + await client.disconnect() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 7a54cece39..e8bd946b79 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,8 +434,8 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,place,prepare,attach} ...", - "positional arguments: {credential,limit,which,remove,place,prepare,attach} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve.", + "usage: loom cell [-h] {credential,limit,which,remove,place,prepare,attach,update} ...", + "positional arguments: {credential,limit,which,remove,place,prepare,attach,update} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. place Place an org in this cell and record its tenancy. prepare Register this cell in the router's directory and write its rows. attach Attach the prepared first cell: place its orgs, serve. update Reserve configuration, finish or revert its rollout, or refresh verified addresses.", "options: -h, --help show this help message and exit" ], "loom cell credential": [ @@ -467,6 +467,11 @@ "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], + "loom cell update": [ + "usage: loom cell update [-h] --cell CELL [--config CONFIG | --finish | --revert | --probe-url PROBE_URL] [--router ROUTER] [--wait WAIT]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --config CONFIG Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; does not install the file --finish Promote next after durable local narrowing and permanent participant turnover --revert Retain current after durable local narrowing and permanent participant turnover; keep placements --probe-url PROBE_URL Later cell's verified probe listener URL; no flag refreshes the first cell's router-configured addresses --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL) --wait WAIT Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)", + "Finish/revert require foundation-capable runtimes and completed positive legacy-process turnover. A census cannot verify that external inventory. Address refresh requires every router updated first; retain old endpoints until every router refreshes or restarts after the directory change." + ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", "options: -h, --help show this help message and exit" diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 52be7c60d8..c17fca77b0 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13233,6 +13233,173 @@ "metavar": null } ] + }, + "update": { + "prog": "loom cell update", + "usage": null, + "description": null, + "epilog": "Finish/revert require foundation-capable runtimes and completed positive legacy-process turnover. A census cannot verify that external inventory. Address refresh requires every router updated first; retain old endpoints until every router refreshes or restarts after the directory change.", + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell_update:cmd_update" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell", + "config", + "finish", + "revert", + "probe_url", + "router", + "wait" + ] + } + ], + "mutually_exclusive": [ + { + "required": false, + "dests": [ + "config", + "finish", + "revert", + "probe_url" + ] + } + ], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "This cell's name (LOOM_CELL)", + "metavar": null + }, + { + "option_strings": [ + "--config" + ], + "dest": "config", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": "pathlib:Path", + "required": false, + "help": "Nonsecret version: 1 settings YAML overlaid on this CLI's effective settings; does not install the file", + "metavar": null + }, + { + "option_strings": [ + "--finish" + ], + "dest": "finish", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Promote next after durable local narrowing and permanent participant turnover", + "metavar": null + }, + { + "option_strings": [ + "--revert" + ], + "dest": "revert", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Retain current after durable local narrowing and permanent participant turnover; keep placements", + "metavar": null + }, + { + "option_strings": [ + "--probe-url" + ], + "dest": "probe_url", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "Later cell's verified probe listener URL; no flag refreshes the first cell's router-configured addresses", + "metavar": null + }, + { + "option_strings": [ + "--router" + ], + "dest": "router", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": false, + "help": "The router's URL (default: LOOM_CELL_ROUTER_URL)", + "metavar": null + }, + { + "option_strings": [ + "--wait" + ], + "dest": "wait", + "action": "_StoreAction", + "default": 600, + "const": null, + "choices": null, + "nargs": null, + "type": "builtins:float", + "required": false, + "help": "Seconds to wait for probe readiness or participant turnover (0 to 86400, default: 600)", + "metavar": null + } + ] } }, "aliases_of": { @@ -13242,7 +13409,8 @@ "remove": "loom cell remove", "place": "loom cell place", "prepare": "loom cell prepare", - "attach": "loom cell attach" + "attach": "loom cell attach", + "update": "loom cell update" }, "choices_help": [ { @@ -13279,6 +13447,11 @@ "dest": "attach", "help": "Attach the prepared first cell: place its orgs, serve.", "metavar": "attach" + }, + { + "dest": "update", + "help": "Reserve configuration, finish or revert its rollout, or refresh verified addresses.", + "metavar": "update" } ] } diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 2e1fc087ee..4970b979b9 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -19,6 +19,7 @@ ROOT = Path(__file__).resolve().parents[2] _CONNECTS = { "run_connected", + "run_control_connected", "connect_checked", "connect_declared", "check_acceptance", diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 60bf88d274..e453db394f 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -102,6 +102,7 @@ def test_cell_control_migrations_are_classified(): # Entries name every sink, including nested callbacks and dynamic table expressions. # A new sink, including another statement in an existing owner, requires classification. GUARDED_WRITERS = { + "server/cli/cell_update_pg.py": {("write.change", "UPDATE", "loom.cell_self"): 1}, "server/cli/cell_attach_pg.py": {("_mode_transaction", "UPDATE", "loom.tenancy_mode"): 1}, "server/cli/cell_control.py": { ("backout.clear_state", "ALTER TABLE", "loom.cell_first"): 3, @@ -114,6 +115,7 @@ def test_cell_control_migrations_are_classified(): ("_write_rows.write", "INSERT INTO", "loom.cell_self"): 1, }, "server/router/directory.py": { + ("update_addresses.change", "UPDATE", "loom_router.cells"): 1, ("set_limit.change", "UPDATE", "loom_router.cells"): 1, ("rotate_credential.change", "UPDATE", "loom_router.cells"): 1, ("claim.change_locked", "UPDATE", "loom_router.cells"): 1, @@ -310,6 +312,39 @@ def function(path: str, name: str) -> ast.AST: @pytest.mark.parametrize( ("path", "name", "required"), [ + ( + "server/cli/cell_update_run.py", + "update", + ['cell_control.run("update", run.start)'], + ), + ( + "server/cli/cell_update_run.py", + "Run.start", + [ + "cell_control.claim(owner.conn, self.call, self.row)", + "local.paired(self.state, self.row)", + ], + ), + ( + "server/cli/cell_update_pg.py", + "write", + ["owner.transaction(change)"], + ), + ( + "server/cli/cell_update_pg.py", + "write.change", + ["cell_control.paired(conn, registered)", "read(conn, expected.cell)"], + ), + ( + "server/router/directory.py", + "update_addresses", + ["_attach_change(cell_name, expected, change)"], + ), + ( + "server/router/directory.py", + "configuration_placements", + ["_attach_change(cell_name, expected, read)"], + ), ( "server/cli/cell_prepare.py", "prepare", diff --git a/tests/unit/test_cell_update.py b/tests/unit/test_cell_update.py new file mode 100644 index 0000000000..bcb7273e07 --- /dev/null +++ b/tests/unit/test_cell_update.py @@ -0,0 +1,96 @@ +"""The update control command validates its action before connecting or admitting work.""" + +from __future__ import annotations + +import argparse +import importlib + +import pytest + +from server.settings import settings + + +def args(**changes): + return argparse.Namespace( + cell="first", wait=0, config=None, finish=False, revert=False, probe_url=None, **changes + ) + + +@pytest.mark.parametrize("wait", [-1, float("inf"), float("nan"), 86401]) +def test_update_refuses_invalid_wait_before_connect(monkeypatch, wait): + command = importlib.import_module("server.cli.cell_update") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://configured") + values = args() + values.wait = wait + assert "wait" in command._args_problem(values) + + +@pytest.mark.parametrize( + "actions", + [ + {"config": "config.yaml", "finish": True}, + {"finish": True, "revert": True}, + {"probe_url": "http://probe", "revert": True}, + ], +) +def test_update_refuses_ambiguous_actions_before_connect(monkeypatch, actions): + command = importlib.import_module("server.cli.cell_update") + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "pg_dsn", "postgresql://configured") + values = args() + for name, value in actions.items(): + setattr(values, name, value) + assert "one" in command._args_problem(values) + + +@pytest.mark.parametrize("refused", [False, True]) +def test_control_coordinator_verifies_and_gates_without_becoming_a_participant( + monkeypatch, refused +): + from unittest.mock import AsyncMock + + from server.cli import _ch + + events = [] + client = object() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + + async def connect(ch): + assert ch is client + events.append("paired-stores") + + async def acceptance(*, org_free): + assert org_free.kind == "write" + events.append("org-free-gate") + if refused: + raise _ch.OrgNotHeld("wrong cell") + + async def body(ch): + assert ch is client + events.append("body") + return 7 + + async def close(ch): + assert ch is client + events.append("disconnect") + + forbidden = AsyncMock(side_effect=AssertionError("update must not enroll")) + monkeypatch.setattr(_ch, "connect_verified", connect) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(_ch, "disconnect", close) + monkeypatch.setattr("server.insights.app.provider_command.model_command", forbidden) + if refused: + with pytest.raises(_ch.OrgNotHeld, match="wrong cell"): + _ch.run_control_connected(body, org_free=_ch.writes("cell control")) + else: + assert _ch.run_control_connected(body, org_free=_ch.writes("cell control")) == 7 + assert events == [ + "paired-stores", + "org-free-gate", + *([] if refused else ["body"]), + "disconnect", + ] + forbidden.assert_not_called() diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py new file mode 100644 index 0000000000..5465767496 --- /dev/null +++ b/tests/unit/test_cell_update_participants.py @@ -0,0 +1,301 @@ +"""A finalization census never shrinks by expiry or accepts an old serving identity.""" + +from __future__ import annotations + +import importlib +from uuid import UUID + +import pytest + + +def participant(n, state="running", proof="", version=1, expired=True): + return { + "participant_id": str(UUID(int=n)), + "state": state, + "host_proof": proof, + "version": version, + "expired": expired, + } + + +def test_capture_keeps_expired_drained_and_uncertified_retired(): + module = importlib.import_module("server.cli.cell_update_participants") + values = [participant(1), participant(2, "drained"), participant(3, "retired")] + assert set(module.capture(values)) == {row["participant_id"] for row in values} + + +def test_capture_omits_only_complete_foundation_retirement_certificate(): + module = importlib.import_module("server.cli.cell_update_participants") + good = "host|boot||namespace|1|3|||cell-retirement-v1" + values = [participant(1, "retired", good), participant(2, "retired", "cell-retirement-v1")] + assert set(module.capture(values)) == {str(UUID(int=2))} + + +def test_current_target_in_initial_census_still_must_turn_over(): + module = importlib.import_module("server.cli.cell_update_participants") + row = participant(1, proof="host|boot||namespace|1|3|||cell-retirement-v1") + assert row["participant_id"] in module.capture([row]) + + +def test_census_conflicting_equal_versions_refuse_instead_of_picking_a_replica(): + module = importlib.import_module("server.cli.cell_update_participants") + with pytest.raises(module.cell_control.Refused, match="conflicting"): + module.latest([participant(1), participant(1, "retired")]) + + +@pytest.mark.parametrize("version", [True, "2", 0, -1, None]) +def test_census_refuses_unreadable_versions(version): + from server.cli import cell_control + from server.cli import cell_update_participants as module + + with pytest.raises(cell_control.Refused, match="unreadable"): + module.latest([participant(1, version=version)]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "change", ["missing", "drained", "expired", "uncertified", "old-version", "other-proof"] +) +async def test_retirement_requires_permanent_exact_identity_and_newer_version(monkeypatch, change): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + initial = participant(1, proof=proof, version=2) + current = participant(1, "retired", proof=proof, version=3) + if change == "missing": + current = None + elif change in ("drained", "expired"): + current["state"] = "drained" if change == "drained" else "running" + elif change == "uncertified": + current["host_proof"] = "" + elif change == "old-version": + current["version"] = 2 + else: + current["host_proof"] = proof.replace("|1|3|", "|2|3|") + monkeypatch.setattr(module, "proves_stopped", lambda _: False) + ch = type("CH", (), {"query_one": AsyncMock(return_value={"running": 0})})() + assert not await module.stopped(ch, initial, current) + ch.query_one.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_exact_retirement_certificate_is_accepted_without_age_or_query_heuristic(): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + ch = type("CH", (), {"query_one": AsyncMock(side_effect=AssertionError("no heuristic"))})() + assert await module.stopped( + ch, participant(1, proof=proof), participant(1, "retired", proof, 2) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "running", [None, {}, {"running": "0"}, {"running": False}, {"running": 1}, {"running": 0}] +) +async def test_physical_death_also_requires_known_zero_strict_query_census(monkeypatch, running): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + monkeypatch.setattr(module, "proves_stopped", lambda _: True) + ch = type("CH", (), {"query_one": AsyncMock(return_value=running)})() + assert await module.stopped(ch, participant(1), None) is ( + running == {"running": 0} and type(running["running"]) is int + ) + assert "skip_unavailable_shards=0" in ch.query_one.call_args.args[0] + + +@pytest.mark.asyncio +async def test_physical_death_query_failure_never_falls_back_to_local(monkeypatch): + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + + monkeypatch.setattr(module, "proves_stopped", lambda _: True) + ch = type("CH", (), {"query_one": AsyncMock(side_effect=OSError("replica unavailable"))})() + with pytest.raises(OSError, match="replica unavailable"): + await module.stopped(ch, participant(1), None) + assert ch.query_one.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "change", + [ + "valid", + "worker", + "captured", + "retired", + "unmarked", + "wrong-cell", + "old-fingerprint", + "no-instance", + "no-participant", + "not-router", + ], +) +async def test_replacement_joins_serving_instance_and_guarded_new_participant(monkeypatch, change): + import json + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.core import cell + + identity = str(UUID(int=2)) + target = cell.as_value({"deployment_id": "dep"}) + proof = "host|boot||namespace|1|3|||cell-retirement-v1" + rows = {identity: participant(2, proof=proof)} + captured = {str(UUID(int=1))} + live = [{"instance_id": "replacement", "serving": change != "worker"}] + value = { + "participant_id": identity, + "fields": target["fields"], + "serves": True, + "marked": True, + "router": True, + } + if change == "captured": + captured.add(identity) + elif change == "retired": + rows[identity]["state"] = "retired" + elif change == "unmarked": + value["marked"] = False + elif change == "not-router": + value["router"] = False + elif change == "old-fingerprint": + value["fields"] = {"deployment_id": "old"} + elif change == "no-instance": + live = [] + elif change == "no-participant": + rows = {} + beat = { + "instance_id": "replacement", + "cell": "other" if change == "wrong-cell" else "first", + "tenancy_mode": "enforced", + "fingerprint": json.dumps(value), + } + monkeypatch.setattr(module.instances, "read_live", AsyncMock(return_value=live)) + ch = SimpleNamespace(query_all=AsyncMock(return_value=[beat])) + run = SimpleNamespace(args=SimpleNamespace(cell="first"), row={"is_first": True}) + assert await module.replacement(ch, run, rows, captured, target) is (change == "valid") + assert "use_query_cache = '0'" in ch.query_all.call_args.args[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "engine", ["ReplacingMergeTree", "ReplicatedReplacingMergeTree", "SharedReplacingMergeTree"] +) +async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.cli.cell_ch import Topology + + events = [] + topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) + monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) + monkeypatch.setattr( + module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + ) + monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + + async def per_host(ch, top, names, sql, params): + if "source:tables" in sql: + return {name: [{"engine": engine}] for name in names} + return { + name: [ + { + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": name, + "replica_is_active": {"a": 1, "b": 1}, + } + ] + for name in names + } + + async def sync(*args): + events.append("sync") + + async def query(sql, params): + events.append("read") + assert "clusterAllReplicas" in sql + assert "skip_unavailable_shards=0" in sql + assert "result_overflow_mode = 'throw'" in sql + assert "use_query_cache = '0'" in sql + assert f"select_sequential_consistency={int(engine.startswith('Shared'))}" in sql + return [participant(1), participant(2), participant(1, "drained", version=2)] + + monkeypatch.setattr(module, "per_host", per_host) + monkeypatch.setattr(module.cell_scan, "_system", sync) + result = await module.census(SimpleNamespace(query_all=query)) + assert len(result) == 2 and result[str(UUID(int=1))]["state"] == "drained" + assert events == (["sync", "read"] if engine.startswith("Replicated") else ["read"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "problem", + ["unlisted", "shards", "missing-table", "unreachable", "extra-replica", "sync-failure"], +) +async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkeypatch, problem): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.cli import cell_update_participants as module + from server.cli.cell_ch import Topology + + topology = Topology( + "replicas", + 0 if problem == "unlisted" else 2 if problem == "shards" else 1, + () if problem == "unlisted" else (("a", 9000), ("b", 9000)), + ) + monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) + monkeypatch.setattr( + module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + ) + monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + + async def per_host(*args): + if problem == "unreachable": + raise OSError("replica unavailable") + if "source:tables" in args[3]: + return { + "a": [{"engine": "ReplicatedReplacingMergeTree"}], + "b": [] + if problem == "missing-table" + else [{"engine": "ReplicatedReplacingMergeTree"}], + } + return { + name: [ + { + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": name, + "replica_is_active": { + "a": 1, + "b": 1, + **({"unknown": 1} if problem == "extra-replica" else {}), + }, + } + ] + for name in ("a", "b") + } + + monkeypatch.setattr(module, "per_host", per_host) + monkeypatch.setattr( + module.cell_scan, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) + ) + query = AsyncMock(side_effect=AssertionError("must not read partial census")) + with pytest.raises((module.cell_control.Refused, OSError)): + await module.census(SimpleNamespace(query_all=query)) + query.assert_not_awaited() diff --git a/tests/unit/test_cell_update_probe.py b/tests/unit/test_cell_update_probe.py new file mode 100644 index 0000000000..fdcdb7d73a --- /dev/null +++ b/tests/unit/test_cell_update_probe.py @@ -0,0 +1,74 @@ +"""A configuration or address proof cannot leave an unbounded HTTP waiter.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import cell_control, cell_update_probe +from tests.unit.test_cell_update_run import row + + +@pytest.mark.asyncio +async def test_update_proof_has_whole_reply_deadline_and_exact_nonce_cleanup(monkeypatch): + class Direct: + conn = object() + + async def call(self, work, *, total_s): + return await work() + + nonce = {"postgres": "pg", "clickhouse": "ch", "bucket": None} + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + monkeypatch.setattr(cell_update_probe.cell_prepare, "_new_nonces", lambda: nonce) + monkeypatch.setattr(cell_update_probe.cell_prepare, "_write_nonces", AsyncMock()) + cleanup = AsyncMock() + monkeypatch.setattr(cell_update_probe.cell_prepare, "_delete_nonces", cleanup) + monkeypatch.setattr(cell_update_probe, "ANSWER_S", 0.02, raising=False) + cancelled = asyncio.Event() + + async def stalled(*args, **kwargs): + try: + await asyncio.Future() + finally: + cancelled.set() + + monkeypatch.setattr(cell_update_probe.cell_prepare, "_register", stalled) + run = SimpleNamespace( + owner=Direct(), + args=SimpleNamespace(wait=0), + ch=object(), + router=None, + transport=None, + row=row(), + ) + task = asyncio.create_task(cell_update_probe.post(run, "/configuration", {})) + done, _ = await asyncio.wait({task}, timeout=0.15) + if not done: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + pytest.fail("update proof exceeded its whole reply deadline") + with pytest.raises(TimeoutError): + await task + assert cancelled.is_set() + cleanup.assert_awaited_once_with(run.ch, nonce) + + +@pytest.mark.asyncio +async def test_router_without_update_route_names_the_required_upgrade(): + import httpx + + from server.cli import cell_prepare + + with pytest.raises(cell_prepare.Refused, match="not upgraded for cell update"): + await cell_prepare._register( + SimpleNamespace(cell="first", wait=0), + httpx.URL("http://router"), + {}, + httpx.MockTransport(lambda _: httpx.Response(404, text="Not Found")), + path="/configuration", + command="update", + ) diff --git a/tests/unit/test_cell_update_run.py b/tests/unit/test_cell_update_run.py new file mode 100644 index 0000000000..7747341ad2 --- /dev/null +++ b/tests/unit/test_cell_update_run.py @@ -0,0 +1,231 @@ +"""Cross-store update ordering and rerun choices use only recorded fingerprints.""" + +from __future__ import annotations + +from argparse import Namespace +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import httpx +import pytest + +from server.cli import cell_control, cell_update_participants, cell_update_pg, cell_update_run +from server.core import cell + + +def value(issuer="old", *, scim=False, orgs=()): + return cell.as_value( + { + "deployment_id": "dep-first", + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + "bearer_issuers": [issuer], + "configured_orgs": list(orgs), + "scim": scim, + } + ) + + +def row(current=None, pending=None): + return { + "cell": "first", + "is_first": True, + "host": "first.example.test", + "deployment_id": "dep-first", + "bucket": "", + "bucket_prefix": "", + "probe_url": "http://probe", + "upstream_url": "http://main", + "current_fp": current, + "next_fp": pending, + "mutation_generation": str(UUID(int=1)), + "state": "serving", + "mode": "enforced", + } + + +@pytest.fixture +def coordinator(monkeypatch): + events = [] + run = cell_update_run.Run( + None, + Namespace(cell="first", config=None, finish=False, revert=False, probe_url=None, wait=0), + httpx.URL("http://router"), + None, + ) + run.owner = SimpleNamespace(conn=object()) + run.row = row(value(), value("new", orgs=("held",))) + run.state = cell_update_pg.State("dep-first", True, "first", "enforced", value(), None, False) + run.target = None + + async def held(current, pending): + events.append("placement-proof") + return ["held"] + + async def write(owner, expected, registered, current, pending, placed): + events.append("local-write") + assert registered is run.row + assert placed == ["held"] or registered["current_fp"] is None + run.state = replace(expected, current=current, pending=pending) + return run.state + + async def durable(): + events.append("durable") + return run.state + + async def publish(target, action): + events.append("directory-" + action) + current, pending = cell_update_run.fingerprints.configuration_pair( + run.row["current_fp"], run.row["next_fp"], target, action + ) + return run.row | {"current_fp": current, "next_fp": pending} + + async def settle(selected, target): + events.append("turnover") + assert selected is run + assert run.state.current == target and run.state.pending is None + assert run.row["next_fp"] is not None + + async def call(path): + events.append("directory-read") + assert path == "" + return run.row + + monkeypatch.setattr(run, "held_orgs", held) + monkeypatch.setattr(run, "durable", durable) + monkeypatch.setattr(run, "publish", publish) + monkeypatch.setattr(run, "call", call) + monkeypatch.setattr(cell_update_pg, "write", write) + monkeypatch.setattr(cell_update_participants, "settle", settle) + return run, events + + +@pytest.mark.asyncio +async def test_config_reserves_directory_before_any_local_acceptance(coordinator): + run, events = coordinator + run.args.config, run.target = Path("target"), value("new", orgs=("held",)) + await run.configuration() + assert events == ["directory-config", "placement-proof", "local-write", "durable"] + assert run.state.pending == run.target + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["finish", "revert"]) +@pytest.mark.parametrize("already_narrowed", [False, True]) +async def test_finalization_repairs_then_cuts_off_then_turns_over_then_releases( + coordinator, action, already_narrowed +): + run, events = coordinator + setattr(run.args, action, True) + target = run.row["next_fp"] if action == "finish" else run.row["current_fp"] + if already_narrowed: + run.state = replace(run.state, current=target, pending=None) + await run.configuration() + assert events == [ + "placement-proof", + "local-write", + "durable", + "turnover", + "durable", + "directory-read", + "directory-" + action, + ] + assert run.state.current == target and run.state.pending is None + assert run.row["current_fp"] == target and run.row["next_fp"] is None + + +@pytest.mark.asyncio +async def test_cutoff_timeout_keeps_directory_dual_without_compensating_local_rewiden( + coordinator, monkeypatch +): + run, events = coordinator + run.args.finish = True + monkeypatch.setattr( + cell_update_participants, "settle", AsyncMock(side_effect=TimeoutError("busy")) + ) + before = run.row.copy() + with pytest.raises(TimeoutError, match="busy"): + await run.configuration() + assert run.row == before + assert run.state.current == before["next_fp"] and run.state.pending is None + assert events == ["placement-proof", "local-write", "durable"] + + +@pytest.mark.asyncio +async def test_prebegin_never_reads_or_places_configured_orgs(coordinator, monkeypatch): + run, events = coordinator + run.row = row() + run.args.config, run.target = Path("target"), value("new", orgs=("unplaced",)) + forbidden = AsyncMock(side_effect=AssertionError("directory reservation before begin")) + monkeypatch.setattr(run, "held_orgs", forbidden) + monkeypatch.setattr(run, "publish", forbidden) + await run.configuration() + assert events == ["local-write", "durable"] + assert run.state.pending == run.target + + +@pytest.mark.asyncio +async def test_lost_generation_after_barrier_never_releases_reservation(coordinator, monkeypatch): + run, events = coordinator + run.args.finish = True + monkeypatch.setattr( + run, "call", AsyncMock(return_value=run.row | {"mutation_generation": str(UUID(int=2))}) + ) + with pytest.raises(cell_control.Refused, match="ownership or reservation changed"): + await run.configuration() + assert "directory-finish" not in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["config", "finish", "revert"]) +async def test_singleton_noop_proves_local_durability_without_turnover(coordinator, action): + run, events = coordinator + run.row["next_fp"] = None + run.state = replace(run.state, current=run.row["current_fp"], pending=None) + if action == "config": + run.args.config, run.target = Path("target"), run.row["current_fp"] + else: + setattr(run.args, action, True) + await run.configuration() + assert events.count("durable") == 1 + assert "turnover" not in events + assert "directory-" + action in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["config", "finish"]) +async def test_scim_on_refuses_before_generation_claim_or_acceptance( + coordinator, monkeypatch, action +): + run, _ = coordinator + unsafe = value("new", scim=True) + run.row["next_fp"] = unsafe + if action == "config": + run.args.config = Path("target") + monkeypatch.setattr(cell_update_run.cell_update_config, "read_target", lambda *_: object()) + monkeypatch.setattr( + cell_update_run.cell_update_config, "target_fingerprint", lambda *_: unsafe + ) + else: + run.args.finish = True + claim = AsyncMock(side_effect=AssertionError("must not claim")) + monkeypatch.setattr(cell_control, "claim", claim) + with pytest.raises(ValueError, match="SCIM"): + await run.start(run.owner) + claim.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_revert_repairs_tenancy_for_pending_placements_before_dropping_fingerprint( + coordinator, monkeypatch +): + run, _ = coordinator + run.args.revert = True + held = AsyncMock(return_value=["held"]) + monkeypatch.setattr(run, "held_orgs", held) + before = run.row.copy() + await run.configuration() + held.assert_awaited_once_with(before["current_fp"], before["next_fp"]) From 9dd0934637c2cb0555939578cc65dc4cee63b18e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:10:47 +0000 Subject: [PATCH 079/181] WIP: Preserve selected driver errors through artifact diagnostics Secondary reporting failures retain the selected work or cleanup failure. Required output failures still refuse an otherwise successful run. The internal artifact-close matrix and final focused gates remain pending. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 20 +- .../test_cell_trigger_gate_diagnostics.py | 233 ++++++++++++++++++ 2 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_diagnostics.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 17c0230ef7..15f9c258ff 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1340,9 +1340,8 @@ async def _run( except BaseException as error: try: artifact.emit("cleanup_error", error_type=type(error).__name__) - except (OSError, ValueError): - if prior is None: - raise + except BaseException: # noqa: BLE001 — diagnostics cannot replace work or cleanup failure. + pass if prior is None: raise if interrupted and prior is None: @@ -1449,18 +1448,19 @@ async def run(ch: Any) -> dict[str, Any]: artifact.emit( "invalid", phase=phase, reason=reason, error_type=type(error).__name__ ) - except (OSError, ValueError): + except BaseException: # noqa: BLE001 — secondary output cannot replace INVALID. pass finally: if artifact is not None: try: artifact.close() - except (OSError, ValueError) as error: - verdict = { - "verdict": "INVALID", - "error_type": type(error).__name__, - "phase": "artifact_close", - } + except BaseException as error: # noqa: BLE001 — required close fails without losing INVALID. + if verdict["verdict"] != "INVALID": + verdict = { + "verdict": "INVALID", + "error_type": type(error).__name__, + "phase": "artifact_close", + } print(json.dumps(verdict, sort_keys=True, allow_nan=False), flush=True) return 0 if verdict["verdict"] == "PASS" else 1 diff --git a/tests/unit/test_cell_trigger_gate_diagnostics.py b/tests/unit/test_cell_trigger_gate_diagnostics.py new file mode 100644 index 0000000000..b2f4594ef8 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_diagnostics.py @@ -0,0 +1,233 @@ +"""Artifact diagnostics retain the first driver failure at the actual owner.""" + +import argparse +import asyncio +import json +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from server.cli import cell_attach_pg as apg + + +@pytest.fixture +def run_driver(monkeypatch, tmp_path): + from server.core import background + + def configure(*, prior=None, cleanup=None, output=None, event="cleanup_error"): + events = [] + + class Conn: + async def close(self, **kwargs): + events.append("public close") + + def terminate(self): + events.append("abort") + + class Triggers: + tables = ("memory_current",) + invariant = org = "digest" + + async def set(self, *args): + pass + + async def restore(self, *args): + events.append("restoration") + if cleanup is not None: + raise cleanup + + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + original = artifact.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if kind == event and output is not None: + raise output + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=Conn())) + monkeypatch.setattr(gate, "_proof", AsyncMock(return_value={})) + monkeypatch.setattr(gate.Triggers, "capture", AsyncMock(return_value=Triggers())) + monkeypatch.setattr(gate, "_memory_set", AsyncMock(side_effect=prior)) + if prior is None: + monkeypatch.setattr(gate, "_arm_order", lambda: ()) + monkeypatch.setattr(gate, "_statistics", lambda samples: {}) + monkeypatch.setattr(background, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "flush_registered_buffers", AsyncMock(return_value=True)) + + async def run(): + try: + return await gate._run( + argparse.Namespace(pg_dsn="unused"), None, artifact, object() + ) + finally: + artifact.close() + + return run, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +async def test_outer_cleanup_diagnostic_preserves_selected_first_error( + run_driver, prior_kind, output_kind +): + prior = prior_kind("work failed first") if prior_kind is not None else None + cleanup = ValueError("cleanup failed") + output = output_kind("diagnostic failed") if output_kind is not None else None + run, events = run_driver(prior=prior, cleanup=cleanup, output=output) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — assert the actual owner failure. + assert error is (prior if prior is not None else cleanup) + assert events.count("cleanup_error") == 1 + assert events.index("public close") < events.index("cleanup_error") + return + raise AssertionError("selected failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event", ["provisional_statistics", "triggers_restored"]) +@pytest.mark.parametrize( + "output_kind", [OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +async def test_required_output_failure_after_successful_work_still_fails( + run_driver, event, output_kind +): + first = output_kind("required artifact failed") + run, events = run_driver(output=first, event=event) + + async def owner(): + try: + await run() + except BaseException as error: # noqa: BLE001 — native errors stay in their owner. + assert error is first + assert "restoration" in events and "public close" in events + return + raise AssertionError("required output failure was swallowed") + + assert await apg._work_result(owner) is None + + +@pytest.mark.asyncio +async def test_successful_driver_and_cleanup_output_returns_normally(run_driver): + run, events = run_driver() + assert await run() == {} + assert events == ["provisional_statistics", "restoration", "triggers_restored", "public close"] + + +@pytest.mark.parametrize("phase", ["invalid", "close"]) +@pytest.mark.parametrize( + "prior_kind", [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "output_kind", + [None, OSError, RuntimeError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_main_preserves_first_invalid_and_requires_successful_artifact_close( + monkeypatch, tmp_path, capsys, phase, prior_kind, output_kind +): + import sys + + import server.settings + import server.wiring + from server.cli import _ch + from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + + fixture, config = fixture_config() + prior = prior_kind("first native work failure") if prior_kind is not None else None + output = output_kind("secondary artifact failure") if output_kind is not None else None + events = [] + real_artifact = gate.Artifact + + class Artifact(real_artifact): + def __init__(self, path): + super().__init__(path) + original = self.file + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if phase == "invalid" and kind == "invalid" and output is not None: + raise output + return original.write(line) + + def close(self): + events.append("file closed") + return original.close() + + def __getattr__(self, name): + return getattr(original, name) + + self.file = File() + + argv = ["driver"] + for key, value in vars( + args(fixture_manifest="unused", output=str(tmp_path / "raw.jsonl")) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: None) + monkeypatch.setattr(gate.Fixture, "read", lambda _: fixture) + monkeypatch.setattr(gate, "_source", lambda _: {}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "Artifact", Artifact) + + def native(*_args, **_kwargs): + if prior is not None: + raise prior + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + def sync(_fd): + assert capsys.readouterr().out == "" + events.append("file sync") + if phase == "close" and output is not None: + raise output + + monkeypatch.setattr(_ch, "run_connected", native) + monkeypatch.setattr(gate.os, "fsync", sync) + invalid = prior is not None or (phase == "close" and output is not None) + try: + result = gate.main() + except BaseException as error: # noqa: BLE001 — an escaped fatal must fail this case only. + pytest.fail(f"main escaped {type(error).__name__}") + assert result == int(invalid) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("INVALID" if invalid else "PASS") + if prior is not None: + assert verdict["error_type"] == prior_kind.__name__ + assert verdict["phase"] == "native_driver" + elif invalid: + assert verdict["error_type"] == output_kind.__name__ + assert verdict["phase"] == "artifact_close" + assert events[-2:] == ["file sync", "file closed"] + assert not any( + json.loads(line).get("verdict") == "PASS" + for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ) From 15f842af44fb49d67a320877f479bfbceac38c44 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:21:26 +0000 Subject: [PATCH 080/181] Preserve artifact sync failures through file close A secondary file-close error must retain the first flush or sync failure. A first close failure after successful sync still fails the run. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 8 ++- .../test_cell_trigger_gate_artifact_close.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_cell_trigger_gate_artifact_close.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 15f9c258ff..ed7f59ec54 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -400,7 +400,13 @@ def close(self) -> None: try: self.file.flush() os.fsync(self.file.fileno()) - finally: + except BaseException: # Retain the flush or sync failure through final close. + try: + self.file.close() + except BaseException: # noqa: BLE001 — close cannot replace failed artifact sync. + pass + raise + else: self.file.close() diff --git a/tests/unit/test_cell_trigger_gate_artifact_close.py b/tests/unit/test_cell_trigger_gate_artifact_close.py new file mode 100644 index 0000000000..3234dc45f5 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_artifact_close.py @@ -0,0 +1,60 @@ +"""Artifact close retains its first flush, sync or file-close failure.""" + +import asyncio + +import pytest + +from bench import cell_trigger_gate as gate + + +@pytest.mark.parametrize("phase", ["flush", "fsync"]) +@pytest.mark.parametrize( + "first_kind", [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +) +@pytest.mark.parametrize( + "second_kind", + [None, OSError, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt], +) +def test_actual_artifact_close_retains_first_flush_sync_or_close_failure( + monkeypatch, tmp_path, phase, first_kind, second_kind +): + first = first_kind("flush or sync failed first") if first_kind is not None else None + second = second_kind("file close failed second") if second_kind is not None else None + artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) + artifact.emit("control") + original = artifact.file + events = [] + + class File: + def flush(self): + events.append("flush") + if phase == "flush" and first is not None: + raise first + return original.flush() + + def close(self): + events.append("file close") + original.close() + if second is not None: + raise second + + def __getattr__(self, name): + return getattr(original, name) + + def sync(_fd): + events.append("fsync") + if phase == "fsync" and first is not None: + raise first + + artifact.file = File() + monkeypatch.setattr(gate.os, "fsync", sync) + selected = first if first is not None else second + if selected is None: + artifact.close() + else: + with pytest.raises(BaseException) as observed: + artifact.close() + assert observed.value is selected + assert original.closed and events[-1] == "file close" + assert events.count("file close") == 1 + assert ("fsync" in events) == (phase != "flush" or first is None) From 361bbbb9e3fcbcbc4af3a8ffd8b6b1c38b83a930 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:33:07 +0000 Subject: [PATCH 081/181] Bound placement exchanges and reserve both request bodies Signed-off-by: zlareb1 --- server/cli/cell_place.py | 12 +- server/router/directory.py | 14 +- tests/unit/test_cell_place.py | 4 +- tests/unit/test_cell_place_envelopes.py | 134 +++++++++++++++++ tests/unit/test_cell_place_http_deadline.py | 157 ++++++++++++++++++++ tests/unit/test_router_attach.py | 25 ++++ 6 files changed, 335 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_cell_place_envelopes.py create mode 100644 tests/unit/test_cell_place_http_deadline.py diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index 573e0ff2fc..c0cc15aeb2 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -19,6 +19,9 @@ from server.router.directory import PLACE_MAX_S from server.settings import settings +# The directory statement bound plus connection and client cleanup. +REQUEST_S = PLACE_MAX_S + 10.0 # not a setting: derived + class PlacementRefused(Exception): """A placement cannot complete with this configuration or directory winner.""" @@ -71,7 +74,7 @@ async def _directory( url, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, - timeout=httpx.Timeout(PLACE_MAX_S + 10.0, connect=5.0), + timeout=httpx.Timeout(REQUEST_S, connect=5.0), ) posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) if posted is None: @@ -88,7 +91,7 @@ async def _directory( async def place(args: argparse.Namespace, router: httpx.URL) -> dict[str, str]: """Hold the operator lock through directory placement and an acknowledged local commit. - Every local phase, including opening and closing the direct session, has a deadline.""" + Local phases and the complete directory exchange have separate deadlines.""" owner = Owner(exit_s=_bound_s()) async def opening() -> None: @@ -113,7 +116,10 @@ async def initial() -> tuple[str, dict[str, Any]]: async def work() -> dict[str, str]: deployment, fingerprint = await owner.call(initial, total_s=_bound_s()) - await _directory(router, args.cell, args.org, deployment, fingerprint) + await owner.call( + lambda: _directory(router, args.cell, args.org, deployment, fingerprint), + total_s=REQUEST_S, + ) async def local() -> None: conn = owner.conn diff --git a/server/router/directory.py b/server/router/directory.py index 8ee6fff3df..93dadd802f 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -859,12 +859,14 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> _PLACEMENT_ORG_MAX = 6 * cell_state.ORG_MAX_BYTES # not a setting: derived -def _placement_refusal(cell: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" +def _placement_refusal(cell: str, deployment_id: str, name: str, value: dict[str, Any]) -> str: + """Reserve both live and operator placement bodies with the longest escaped org.""" # The default encoding is the longest: spaces after separators, non-ASCII escaped. - size = len(json.dumps({"org": "", "cell": cell, "fingerprint": value})) + _PLACEMENT_ORG_MAX + envelopes = ( + {"org": "", "cell": cell, "fingerprint": value}, + {"org": "", "deployment_id": deployment_id, "fingerprint": value}, + ) + size = max(len(json.dumps(body)) for body in envelopes) + _PLACEMENT_ORG_MAX if size <= MAX_BODY_BYTES: return "" sizes = sorted(((len(json.dumps(v)), k) for k, v in value["fields"].items()), reverse=True) @@ -890,7 +892,7 @@ def begin_refusal(found: Attaching, begin: Begin) -> str: if value is None: continue if reason := _fingerprint_refusal(found, name, value) or _placement_refusal( - row.cell, name, value + row.cell, row.deployment_id, name, value ): return reason # Compared as JSON, not as Python values, where `false == 0` and `1 == 1.0`: a rerun diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py index 26a06748a1..a8536a27b4 100644 --- a/tests/unit/test_cell_place.py +++ b/tests/unit/test_cell_place.py @@ -486,13 +486,13 @@ async def timed(self, work, *, total_s): return await actual_call(self, work, total_s=total_s) async def directory_call(*args): - # HTTP owns its existing independent transport limit, not a local phase cap. + # The complete HTTP exchange has its independent directory-derived phase. await asyncio.sleep(0.06) monkeypatch.setattr(cell_control.Owner, "call", timed) monkeypatch.setattr(cell_place, "_directory", directory_call) await invoke_place() - assert observed == [0.03, 0.03, 0.03] + assert observed == [0.03, 0.03, cell_place.REQUEST_S, 0.03] conn, _ = local assert conn.committed and conn.closed and not conn.terminated diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py new file mode 100644 index 0000000000..15dd7d779e --- /dev/null +++ b/tests/unit/test_cell_place_envelopes.py @@ -0,0 +1,134 @@ +"""Every accepted fingerprint fits both placement envelopes for every valid org.""" + +from __future__ import annotations + +import hashlib +import json +from types import SimpleNamespace +from uuid import UUID + +import httpx +import pytest +from starlette.requests import Request + +from server.core import cell +from server.router import app, directory +from server.router.limits import MAX_BODY_BYTES +from server.settings import settings + +DEPLOYMENT = str(UUID(int=1)) +TOKEN = "lco_" + "7" * 128 + + +def fingerprint(): + return cell.as_value( + { + **cell.DEFAULTS, + "deployment_id": DEPLOYMENT, + "public_host": "first.example.test", + "oauth_issuer_host": "first.example.test", + } + ) + + +def envelopes(name, value, org=""): + return ( + {"org": org, "cell": name, "fingerprint": value}, + {"org": org, "deployment_id": DEPLOYMENT, "fingerprint": value}, + ) + + +def attaching(name): + row = SimpleNamespace( + cell=name, + is_first=True, + state="attaching", + mode="recording", + deployment_id=DEPLOYMENT, + current_fp=None, + next_fp=None, + host="first.example.test", + ) + return SimpleNamespace(row=row, other=None, attach_started_at=None, bucket="", bucket_prefix="") + + +def begin(value, position): + return directory.Begin( + DEPLOYMENT, + value if position == "current_fp" else fingerprint(), + value if position == "next_fp" else None, + "a" * 64, + {}, + ) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_reserves_operator_envelope_before_accepting_fingerprint(name, position): + value = fingerprint() + live = envelopes(name, value)[0] + # This fits the previous live-only reservation, but not the operator envelope. + live_size = len(json.dumps(live)) + 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - live_size) + reason = directory.begin_refusal(attaching(name), begin(value, position)) + assert position in reason and str(MAX_BODY_BYTES) in reason + assert "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("size", ["default", "boundary"]) +@pytest.mark.parametrize("org", ["acme", "\x01" * cell.ORG_MAX_BYTES]) +async def test_accepted_fingerprint_fits_both_encodings_and_operator_handler( + monkeypatch, name, position, size, org +): + value = fingerprint() + if size == "boundary": + empty_size = max(len(json.dumps(body)) for body in envelopes(name, value)) + value["fields"]["oidc_client_id"] = "x" * ( + MAX_BODY_BYTES - empty_size - 6 * cell.ORG_MAX_BYTES + ) + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + attempt = begin(value, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + bodies = envelopes(name, value, org) + encoded = [ + (json.dumps(body).encode(), httpx.Request("POST", "http://inert.test", json=body).content) + for body in bodies + ] + assert all(len(raw) <= MAX_BODY_BYTES for pair in encoded for raw in pair) + calls = [] + + async def place(actual_org, actual_name, refusal, *, actor, verify): + calls.append((actual_org, actual_name, actor)) + locked = SimpleNamespace( + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + ) + assert verify(locked) == "" + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + for raw in encoded[1]: + scope = { + "type": "http", + "method": "POST", + "path": f"/cells/v1/operator/cells/{name}/place", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(raw=raw): + return {"type": "http.request", "body": raw, "more_body": False} + + response = await app.Router().operator_place(Request(scope, receive)) + assert response.status_code == 200 + assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} + assert calls == [(org, name, "operator")] * 2 diff --git a/tests/unit/test_cell_place_http_deadline.py b/tests/unit/test_cell_place_http_deadline.py new file mode 100644 index 0000000000..79d278903a --- /dev/null +++ b/tests/unit/test_cell_place_http_deadline.py @@ -0,0 +1,157 @@ +"""A complete HTTP exchange cannot outlive its directory-derived owner phase.""" + +from __future__ import annotations + +import asyncio +import json +import time + +import httpcore +import httpx +import pytest +from httpcore._backends.mock import AsyncMockBackend, AsyncMockStream + +from server.cli import cell_control, cell_place +from server.settings import settings +from tests.unit import test_cell_place as helpers + +local = helpers.local +invoke_place = helpers.invoke_place + +_DIRECTORY = cell_place._directory + + +@pytest.fixture +def exchange(local, monkeypatch): + conn, _ = local + events = [] + entered = asyncio.Event() + state = {"mode": "response", "reads": [], "owners": []} + real_close = conn.close + real_call = cell_control.Owner.call + actual_client = httpx.AsyncClient + + async def close(**kwargs): + events.append("pg-close") + await asyncio.sleep(0) + await real_close(**kwargs) + events.append("pg-ack") + + async def timed(owner, work, *, total_s): + if total_s == 0.1: + state["owners"].append(owner) + return await real_call(owner, work, total_s=total_s) + + class Stream(AsyncMockStream): + def __init__(self): + super().__init__([]) + self.headers = False + + async def read(self, max_bytes, timeout=None): # noqa: ASYNC109 — httpcore transport API + if not self.headers: + self.headers = True + if state["mode"] == "response": + return b"HTTP/1.1 200 OK\r\nContent-Length: 100000000\r\n\r\n" + status = b"503 Unavailable" if state["mode"] == "refusal" else b"200 OK" + body = json.dumps( + {"error": "standby stalled", "code": "BACKEND_UNAVAILABLE"} + if state["mode"] == "refusal" + else {"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"} + ).encode() + # A timely reply may exceed the separate local phase allowance. + await asyncio.sleep(0.04) + return ( + b"HTTP/1.1 " + + status + + b"\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + entered.set() + state["reads"].append(timeout) + try: + async with asyncio.timeout(timeout): + await asyncio.sleep(0.002) + return b" " + except asyncio.CancelledError: + events.append("http-cancel") + raise + + class Backend(AsyncMockBackend): + async def connect_tcp(self, *args, **kwargs): + return Stream() + + class Client(actual_client): + async def __aexit__(self, *args): + if state["mode"] == "client-exit": + entered.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + events.append("http-cancel") + raise + return await super().__aexit__(*args) + + def client(**kwargs): + transport = httpx.AsyncHTTPTransport() + transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) + return Client(transport=transport, **kwargs) + + conn.close = close + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "REQUEST_S", 0.1, raising=False) + monkeypatch.setattr(cell_control.Owner, "call", timed) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return conn, events, entered, state + + +@pytest.mark.parametrize("blocking", ["response", "client-exit"]) +@pytest.mark.parametrize("stop", ["deadline", "cancel"]) +async def test_whole_exchange_closes_pg_before_http_cancellation(exchange, blocking, stop): + conn, events, entered, state = exchange + state["mode"] = blocking + started = time.monotonic() + task = asyncio.create_task(invoke_place()) + try: + await asyncio.wait_for(entered.wait(), 0.2) + if stop == "cancel": + task.cancel("operator stop") + done, _ = await asyncio.wait({task}, timeout=0.2) + assert done, "per-read HTTP timeouts cannot bound a continuing response or client cleanup" + with pytest.raises( + TimeoutError if stop == "deadline" else asyncio.CancelledError + ) as caught: + await task + if stop == "cancel": + assert str(caught.value) == "operator stop" + assert time.monotonic() - started < 0.3 + assert conn.closed and not conn.committed and not conn.terminated + assert not any("INSERT" in call for call in conn.calls) + assert events.index("pg-ack") < events.index("http-cancel") + assert events.count("pg-close") == 1 + if stop == "deadline": + (owner,) = state["owners"] + assert owner.close_end == owner.phase_end + if blocking == "response": + assert state["reads"] and all(value == 0.1 for value in state["reads"]) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("mode", ["success", "refusal"]) +async def test_timely_actual_http_reply_keeps_local_phase_bounds(exchange, mode): + conn, events, _, state = exchange + state["mode"] = mode + if mode == "refusal": + with pytest.raises(cell_place.PlacementRefused, match="standby stalled"): + await invoke_place() + assert not conn.committed + else: + assert (await invoke_place())["deployment_id"] == "dep-eu-1" + assert conn.committed + assert conn.closed and not conn.terminated + assert events.count("pg-close") == 1 diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index d9d4b9e7b4..4b0ec1f35c 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1456,3 +1456,28 @@ def test_native_compact_batch_with_control_envelope_fits_public_handler(client, assert response.status_code == 200, response.text assert response.json()["placed"] == len(batch) assert set(store.placements) == set(batch) + + +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +def test_begin_refuses_operator_envelope_overflow_before_probe_or_writes( + client: TestClient, store: _Store, listener: _Listener, position: str +) -> None: + deployment = str(UUID(int=1)) + store.cells["eu-1"]["deployment_id"] = deployment + current = cell.as_value({**FIELDS, "deployment_id": deployment}) + grown = cell.as_value({**FIELDS, "deployment_id": deployment, "oidc_client_id": ""}) + live = {"org": "", "cell": "eu-1", "fingerprint": grown} + available = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + grown["fields"]["oidc_client_id"] = "x" * available + body = ( + _begin(deployment_id=deployment, current_fp=current, **{position: grown}) + if position == "next_fp" + else _begin(deployment_id=deployment, current_fp=grown) + ) + response = _call(client, "POST", "attach/begin", body) + assert response.status_code == 409 + assert position in response.json()["error"] + assert "oidc_client_id" in response.json()["error"] + assert store.statements == directory.ATTACH_READ_STATEMENTS + assert listener.asked == [] and store.audit == [] + assert store.cells["eu-1"]["attach_started_at"] is None From a9580c1e0193bc58f0c1ea089652a542b2e001bf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:49:01 +0000 Subject: [PATCH 082/181] The cell trigger driver retains its first failure through cleanup Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 110 ++++-- tests/unit/test_bench_cell_trigger_gate.py | 6 +- .../test_cell_trigger_gate_first_error.py | 321 ++++++++++++++++++ 3 files changed, 408 insertions(+), 29 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_first_error.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index ed7f59ec54..32803bd38d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -455,7 +455,9 @@ async def _health( ) -> None: from server.settings import settings - async with httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) as client: + context = httpx.AsyncClient(timeout=IO_S, follow_redirects=False, trust_env=False) + client = await context.__aenter__() + try: health = await client.get(f"{args.base_url.rstrip('/')}/v1/health") deployment = await client.get(f"{args.base_url.rstrip('/')}/v1/health/deployment") _require(health.status_code == deployment.status_code == 200, "health HTTP status") @@ -492,6 +494,14 @@ async def _health( ) launch = launch if launch is not None else _launch(args, fixture, settings) _require(h.get("instance") == launch["health_instance"], "serving owned process") + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) def _object(value: Any) -> dict[str, Any]: @@ -602,6 +612,7 @@ async def _proof( async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: conn = ch = None + failure: BaseException | None = None try: conn = await asyncpg.connect( args.pg_dsn, @@ -612,13 +623,22 @@ async def _preflight(args: argparse.Namespace, fixture: Fixture) -> None: ch = await ReadOnlyCH.open() async with asyncio.timeout(IO_S): await _proof(conn, ch, args, fixture) - finally: - try: - if ch is not None: - await ch.close() - finally: - if conn is not None: - await _close_pg(conn) + except BaseException as error: # noqa: BLE001 — cleanup must retain any preflight failure. + failure = error + try: + if ch is not None: + await ch.close() + except BaseException as error: # noqa: BLE001 — both metadata clients must be closed. + if failure is None: + failure = error + try: + if conn is not None: + await _close_pg(conn) + except BaseException as error: # noqa: BLE001 — preserve the first work or close failure. + if failure is None: + failure = error + if failure is not None: + raise failure async def _debt(conn: Any, expected: int) -> None: @@ -681,7 +701,9 @@ async def verify(self, conn: Any) -> None: async def set(self, conn: Any, on: bool) -> None: await self.verify(conn) - async with conn.transaction(): + transaction = conn.transaction() + await transaction.start() + try: for table in self.tables: if table == "tenancy": continue @@ -690,18 +712,34 @@ async def set(self, conn: Any, on: bool) -> None: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') if on: await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() self.on = on await self.verify(conn) async def restore(self, conn: Any) -> None: # Recreate the captured exact org guards even after a partial failed toggle. - async with conn.transaction(): + transaction = conn.transaction() + await transaction.start() + try: for table in self.tables: if table == "tenancy": continue for name in cell_triggers.ORG_TRIGGERS: await conn.execute(f'DROP TRIGGER IF EXISTS {name} ON loom."{table}"') await conn.execute(cell_triggers.create_statement(table, name)) + except BaseException: + try: + await transaction.rollback() + except BaseException: # noqa: BLE001 — rollback cannot replace the DDL failure. + pass + raise + await transaction.commit() self.on = True await self.verify(conn) _require(not await cell_triggers.missing(conn), "restored canonical triggers") @@ -772,12 +810,14 @@ async def _memory_set( ) -> list[RequestRow]: rows = [] prefix = f"gate-{uuid.uuid4().hex}-{arm}" - async with httpx.AsyncClient( + context = httpx.AsyncClient( timeout=IO_S, headers={"Authorization": f"Bearer {args.token}"}, follow_redirects=False, trust_env=False, - ) as client: + ) + client = await context.__aenter__() + try: for i in range(CALLS + WARMUP): key = f"{prefix}-{i}" value = _payload(key) @@ -843,18 +883,29 @@ async def _memory_set( code = candidate except (ValueError, TypeError, AttributeError): pass - emit( - "request_error", - arm=arm, - index=i, - key=key, - measured=i >= WARMUP, - elapsed_ms=elapsed, - status=response.status_code if response is not None else None, - code=code, - error_type=type(error).__name__, - ) + try: + emit( + "request_error", + arm=arm, + index=i, + key=key, + measured=i >= WARMUP, + elapsed_ms=elapsed, + status=response.status_code if response is not None else None, + code=code, + error_type=type(error).__name__, + ) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the request failure. + pass raise + except BaseException as error: + try: + await context.__aexit__(type(error), error, error.__traceback__) + except BaseException: # noqa: BLE001 — HTTP cleanup cannot replace work failure. + pass + raise + else: + await context.__aexit__(None, None, None) return rows @@ -997,6 +1048,7 @@ async def _delivery( results: list[dict[str, Any]] = [] _index = 0 + failure: BaseException | None = None with customer.for_org(args.org, namespace=args.namespace, subject=SUBJECT): started = time.perf_counter_ns() try: @@ -1009,12 +1061,20 @@ async def _delivery( "real worker claimed/published/failure", ) except BaseException as error: - emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + failure = error + try: + emit("worker_error", arm=arm, index=_index, error_type=type(error).__name__) + except BaseException: # noqa: BLE001 — diagnostics cannot replace the worker failure. + pass raise finally: elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 for result_index, outcome in enumerate(results): - emit("worker", arm=arm, index=result_index, **outcome) + try: + emit("worker", arm=arm, index=result_index, **outcome) + except BaseException: # Prior results cannot replace failed work. + if failure is None: + raise _require(math.isfinite(elapsed) and elapsed > 0, "positive worker block elapsed") rps = BATCHES * batch / elapsed emit("delivery_block", arm=arm, elapsed_seconds=elapsed, rows=BATCHES * batch, rps=rps) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 824808214a..a625ab5a27 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,7 +8,6 @@ import json import socket import uuid -from contextlib import asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock @@ -513,9 +512,8 @@ def __init__(self): self.drift = False self.statements = [] - @asynccontextmanager - async def transaction(self): - yield + def transaction(self): + return SimpleNamespace(start=AsyncMock(), rollback=AsyncMock(), commit=AsyncMock()) async def execute(self, sql): self.statements.append(sql) diff --git a/tests/unit/test_cell_trigger_gate_first_error.py b/tests/unit/test_cell_trigger_gate_first_error.py new file mode 100644 index 0000000000..7a3ccdf41b --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_first_error.py @@ -0,0 +1,321 @@ +"""Driver work retains its error through secondary cleanup and artifact failures.""" + +import asyncio +import json +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest +from asyncpg.transaction import Transaction + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + +FIRST = [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] +SECOND = [None, OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +def failure(kind): + return None if kind is None else kind("injected failure") + + +async def observed(work): + try: + await work() + except BaseException as error: # noqa: BLE001 — native errors stay inside this test. + return error + return None + + +@pytest.fixture(autouse=True) +def no_external_io(monkeypatch): + async def forbidden(*_args, **_kwargs): + raise AssertionError("unexpected external I/O") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", forbidden) + + +@pytest.fixture +def artifact_output(monkeypatch, tmp_path): + artifacts = [] + + def configure(errors): + artifact = gate.Artifact(str(tmp_path / f"raw-{len(artifacts)}.jsonl")) + artifacts.append(artifact) + original = artifact.file + events = [] + + class File: + def write(self, line): + kind = json.loads(line)["kind"] + events.append(kind) + if (error := errors.get(kind)) is not None: + raise error + return original.write(line) + + def __getattr__(self, name): + return getattr(original, name) + + artifact.file = File() + return artifact, events + + yield configure + for artifact in artifacts: + artifact.close() + + +@pytest.fixture +def http_edges(monkeypatch): + import server.settings + + fixture, config = fixture_config() + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(gate, "CALLS", 1) + monkeypatch.setattr(gate, "WARMUP", 0) + client_class = httpx.AsyncClient + + def configure(first=None, closing=None, *, bad_status=False): + events = [] + + class Transport(httpx.AsyncBaseTransport): + async def handle_async_request(self, request): + events.append("request") + if first is not None: + raise first + if request.url.path.endswith("/memory.set"): + sent = json.loads(request.content) + body = { + "memory_id": "00000000-0000-0000-0000-000000000001", + "version": 1, + "status": "active", + "memory_key": sent["memory_key"], + "namespace": sent["namespace"], + "content_sha256": gate.sha256_value(sent["value"]), + } + elif request.url.path.endswith("/deployment"): + body = {"deployment_id": fixture.deployment_id} + else: + body = { + "build_commit": fixture.build_commit, + "deployment_config": {"instance_id": fixture.instance_id}, + "ch_ok": True, + "pg_ok": True, + "status": "ok", + "fake_embedding": True, + "fake_llm": True, + "pending_migrations": [], + "storage_authority": "postgres", + "llm_provider": config.llm_provider, + "llm_model": config.llm_model, + "embedding_model": config.embedding_model, + "embedding_dim_configured": config.embedding_dim, + "embedding_dim_probed": config.embedding_dim, + "instance": "test:1", + } + return httpx.Response(503 if bad_status else 200, json=body) + + async def aclose(self): + events.append("http close") + if closing is not None: + raise closing + + def client(**kwargs): + return client_class(**kwargs, transport=Transport()) + + monkeypatch.setattr(gate.httpx, "AsyncClient", client) + return fixture, events + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("ch_kind", SECOND) +@pytest.mark.parametrize("pg_kind", SECOND) +async def test_preflight_retains_first_error_and_attempts_both_closes( + monkeypatch, first_kind, ch_kind, pg_kind +): + first, ch_error, pg_error = map(failure, (first_kind, ch_kind, pg_kind)) + events = [] + + async def proof(*_args): + events.append("proof") + if first is not None: + raise first + + async def close_ch(): + events.append("ch close") + if ch_error is not None: + raise ch_error + + async def close_pg(*_args): + events.append("pg close") + if pg_error is not None: + raise pg_error + + monkeypatch.setattr(gate.asyncpg, "connect", AsyncMock(return_value=object())) + monkeypatch.setattr( + gate.ReadOnlyCH, "open", AsyncMock(return_value=SimpleNamespace(close=close_ch)) + ) + monkeypatch.setattr(gate, "_proof", proof) + monkeypatch.setattr(gate, "_close_pg", close_pg) + assert await observed(lambda: gate._preflight(args(), None)) is (first or ch_error or pg_error) + assert events == ["proof", "ch close", "pg close"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("close_kind", SECOND) +async def test_http_client_exit_retains_work_failure_and_requires_successful_close( + http_edges, consumer, first_kind, close_kind +): + first, closing = failure(first_kind), failure(close_kind) + fixture, events = http_edges(first, closing) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + assert await observed(work) is (first or closing) + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consumer", ["health", "memory"]) +async def test_semantic_http_refusal_survives_client_close_error(http_edges, consumer): + fixture, events = http_edges(closing=OSError("close failed"), bad_status=True) + + async def work(): + if consumer == "health": + await gate._health(args(), fixture, {"health_instance": "test:1"}) + else: + await gate._memory_set(args()) + + field = "health HTTP status" if consumer == "health" else "memory.set HTTP status" + error = await observed(work) + assert isinstance(error, gate.Invalid) and str(error) == field + assert events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["request", "request_record"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_request_diagnostic_retains_request_or_required_record_failure( + http_edges, artifact_output, source, first_kind, output_kind +): + first, output = failure(first_kind), failure(output_kind) + _, http_events = http_edges(first if source == "request" else None) + errors = {"request_error": output} + if source == "request_record": + errors["request"] = first + artifact, events = artifact_output(errors) + assert await observed(lambda: gate._memory_set(args(), emit=artifact.emit)) is first + assert events[-1] == "request_error" and http_events[-1] == "http close" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("record", ["worker_error", "worker"]) +@pytest.mark.parametrize("first_kind", FIRST[1:]) +@pytest.mark.parametrize("output_kind", SECOND) +async def test_worker_diagnostics_and_prior_results_retain_worker_failure( + monkeypatch, artifact_output, record, first_kind, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + first, output = failure(first_kind), failure(output_kind) + good = pg_delivery.DeliveryResult(claimed=1, published=1) + outcomes = [good, first] if record == "worker" else [first] + monkeypatch.setattr(gate, "BATCHES", len(outcomes)) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(side_effect=outcomes) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + artifact, events = artifact_output({record: output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is first + assert worker.await_count == len(outcomes) and "worker_error" in events + if record == "worker": + assert events == ["worker_error", "worker"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_kind", SECOND) +async def test_successful_worker_work_still_requires_result_output( + monkeypatch, artifact_output, output_kind +): + from server.memory.workers import pg_delivery + from server.settings import settings + + output = failure(output_kind) + monkeypatch.setattr(gate, "BATCHES", 1) + monkeypatch.setattr(settings, "pg_delivery_batch_rows", 1) + worker = AsyncMock(return_value=pg_delivery.DeliveryResult(claimed=1, published=1)) + monkeypatch.setattr(pg_delivery, "run_pass", worker) + ticks = iter([0, 1_000_000_000]) + monkeypatch.setattr(gate.time, "perf_counter_ns", lambda: next(ticks)) + artifact, events = artifact_output({"worker": output}) + assert await observed(lambda: gate._delivery(args(), 1, emit=artifact.emit, arm=0)) is output + assert worker.await_count == 1 + assert events == (["worker"] if output is not None else ["worker", "delivery_block"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["set", "restore"]) +@pytest.mark.parametrize("first_kind", FIRST) +@pytest.mark.parametrize("exit_kind", SECOND) +async def test_trigger_work_retains_error_through_transaction_exit_and_owner_cleanup( + monkeypatch, method, first_kind, exit_kind +): + first, exit_error = failure(first_kind), failure(exit_kind) + events = [] + + class Connection: + _pool_release_ctr = 0 + _top_xact = None + _protocol = SimpleNamespace(is_in_transaction=lambda: False) + + def __init__(self): + self.closed = False + + def transaction(self): + return Transaction(self, None, False, False) + + def is_closed(self): + return self.closed + + async def execute(self, sql): + events.append(sql.split()[0]) + if sql.startswith("DROP") and first is not None: + raise first + if sql.startswith(("ROLLBACK", "COMMIT")) and exit_error is not None: + raise exit_error + + async def close(self, **kwargs): + events.append("public close") + self.closed = True + + def terminate(self): + events.append("abort") + self.closed = True + + triggers = gate.Triggers(("memory_current",), "invariant", "org") + monkeypatch.setattr(triggers, "verify", AsyncMock()) + monkeypatch.setattr(gate.cell_triggers, "missing", AsyncMock(return_value=[])) + conn = Connection() + + async def work(): + if method == "set": + await triggers.set(conn, False) + else: + await gate._restore(triggers, conn, time.monotonic() + 1) + + assert await observed(work) is (first or exit_error) + if method == "restore" and (first is not None or exit_error is not None): + assert conn.closed and "public close" in events From 2ff89318f015595fcd034671bbc1e2630317852f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:58:21 +0000 Subject: [PATCH 083/181] Describe trigger creation before the attach mode commit Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c738775756..07115e4b5a 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -789,9 +789,9 @@ read-only scope, lists any tenancy row the directory does not place. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since From ef7e606efcab77d8086d8e3d860aa6d5f888c840 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 02:58:38 +0000 Subject: [PATCH 084/181] Reserve native fingerprint defaults in placement bodies Native placement callers serialize every known fingerprint field. Reserve their defaults when a stored fingerprint omits them, while retaining unknown stored fields in the size bound. Signed-off-by: zlareb1 --- server/router/directory.py | 2 + tests/unit/test_cell_place_envelopes.py | 198 ++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/server/router/directory.py b/server/router/directory.py index 93dadd802f..4eca1b74bd 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -861,6 +861,8 @@ def _fingerprint_refusal(found: Attaching, name: str, value: dict[str, Any]) -> def _placement_refusal(cell: str, deployment_id: str, name: str, value: dict[str, Any]) -> str: """Reserve both live and operator placement bodies with the longest escaped org.""" + # Native callers materialize known defaults; retain unknown stored fields too. + value = {**value, "fields": {**cell_state.DEFAULTS, **value["fields"]}} # The default encoding is the longest: spaces after separators, non-ASCII escaped. envelopes = ( {"org": "", "cell": cell, "fingerprint": value}, diff --git a/tests/unit/test_cell_place_envelopes.py b/tests/unit/test_cell_place_envelopes.py index 15dd7d779e..40b89fabda 100644 --- a/tests/unit/test_cell_place_envelopes.py +++ b/tests/unit/test_cell_place_envelopes.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import itertools import json from types import SimpleNamespace from uuid import UUID @@ -132,3 +133,200 @@ async def receive(raw=raw): assert response.status_code == 200 assert json.loads(response.body) == {"org": org, "cell": name, "deployment_id": DEPLOYMENT} assert calls == [(org, name, "operator")] * 2 + + +OMITTABLE_DEFAULTS = ( + "bearer_issuers", + "bucket", + "bucket_prefix", + "configured_orgs", + "end_user_orgs", + "oidc_issuer", + "scim", +) + + +@pytest.fixture +def native_fingerprint(monkeypatch): + configured = SimpleNamespace( + public_base_url="https://first.example.test", + oauth_issuer="", + oidc_issuer="", + oidc_client_id="", + scim_token="", + document_s3_bucket="", + document_s3_prefix="", + oidc_jit_org="", + self_signup="closed", + self_signup_org="", + code_org="", + tracker_org="", + trusted_issuers=lambda: set(), + end_user_issuers=lambda: {}, + ) + monkeypatch.setattr(cell, "settings", configured) + monkeypatch.setattr(cell, "_code_orgs", None) + + def native(client_id): + configured.oidc_client_id = client_id + # Both native placement callers serialize this complete field set. + return cell.as_value(cell.fingerprint(DEPLOYMENT)) + + return native + + +def omitted_values(): + return ( + subset + for count in range(len(OMITTABLE_DEFAULTS) + 1) + for subset in itertools.combinations(OMITTABLE_DEFAULTS, count) + ) + + +def stored_fingerprint(native, omitted, unknown): + return cell.as_value( + { + **{key: value for key, value in native["fields"].items() if key not in omitted}, + **unknown, + } + ) + + +def accepted_row(name, attempt): + return SimpleNamespace( + cell=name, + deployment_id=DEPLOYMENT, + current_fp=attempt.current_fp, + next_fp=attempt.next_fp, + state="serving", + is_first=True, + mode="enforced", + closed=False, + holds=lambda token: True, + ) + + +async def route_response(monkeypatch, name, attempt, body, operator): + calls = [] + row = accepted_row(name, attempt) + + async def place(org, actual_name, refusal, *, actor, verify=None): + assert actual_name == name and refusal(row) == "" + if verify is not None: + assert verify(row) == "" + calls.append((org, actor)) + return name, DEPLOYMENT + + monkeypatch.setattr(directory, "place", place) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + router = app.Router() + + async def authenticated(request, actual_name): + assert actual_name == name + return "inert-cell-credential" + + monkeypatch.setattr(router, "_authenticated", authenticated) + monkeypatch.setattr(router, "placements", SimpleNamespace(admit=lambda name: True)) + raw = httpx.Request("POST", "http://inert.test", json=body).content + scope = { + "type": "http", + "method": "POST", + "path": "/cells/v1/placements", + "path_params": {"cell": name}, + "headers": [ + (b"authorization", f"Bearer {TOKEN}".encode()), + (b"content-length", str(len(raw)).encode()), + ], + } + + async def receive(): + return {"type": "http.request", "body": raw, "more_body": False} + + handler = router.operator_place if operator else router.place + response = await handler(Request(scope, receive)) + return response, calls, len(raw) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x"}, {"future": "x" * 512}]) +def test_sparse_admission_reserves_matching_native_callers( + native_fingerprint, name, position, unknown +): + org = "\x01" * cell.ORG_MAX_BYTES + for omitted in omitted_values(): + stored = stored_fingerprint(native_fingerprint(""), omitted, unknown) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + stored["fields"]["oidc_client_id"] = client_id + native = native_fingerprint(client_id) + attempt = begin(stored, position) + row = accepted_row(name, attempt) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(row, native) + reason = directory.begin_refusal(attaching(name), attempt) + if not reason: + # Admission must cover what these callers send, even if storage is sparse. + assert all( + len(httpx.Request("POST", "http://inert.test", json=body).content) <= MAX_BODY_BYTES + for body in envelopes(name, native, org) + ), (omitted, unknown) + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +async def test_sparse_boundary_refuses_unusable_native_requests( + monkeypatch, native_fingerprint, name, position, operator +): + stored = stored_fingerprint(native_fingerprint(""), OMITTABLE_DEFAULTS, {}) + empty = max(len(json.dumps(body)) for body in envelopes(name, stored)) + stored["fields"]["oidc_client_id"] = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(stored["fields"]["oidc_client_id"]) + attempt = begin(stored, position) + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + assert cell.named_org(org) == org and not cell.org_name_refusal(org) + body = envelopes(name, native, org)[int(operator)] + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + if org == "acme": + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + else: + assert size > MAX_BODY_BYTES + assert response.status_code == (413 if operator else 400) + assert not calls + reason = directory.begin_refusal(attaching(name), attempt) + assert position in reason and "oidc_client_id" in reason + + +@pytest.mark.parametrize("name", ["a", "a" * 32]) +@pytest.mark.parametrize("position", ["current_fp", "next_fp"]) +@pytest.mark.parametrize("operator", [False, True]) +@pytest.mark.parametrize("unknown", [{}, {"future": "x" * 512}]) +async def test_accepted_sparse_boundary_serves_native_requests( + monkeypatch, native_fingerprint, name, position, operator, unknown +): + native = native_fingerprint("") + # Unknown stored fields keep their existing reservation, although native callers + # do not emit them. Reserve that representation with all known defaults present. + reserved = cell.as_value({**native["fields"], **unknown}) + empty = max(len(json.dumps(body)) for body in envelopes(name, reserved)) + client_id = "x" * (MAX_BODY_BYTES - empty - 6 * cell.ORG_MAX_BYTES) + native = native_fingerprint(client_id) + stored = stored_fingerprint(native, OMITTABLE_DEFAULTS, unknown) + attempt = begin(stored, position) + assert directory.begin_refusal(attaching(name), attempt) == "" + assert cell.matches(native["fields"], stored) + assert app.fingerprint_matches(accepted_row(name, attempt), native) + for org in ("acme", "\x01" * cell.ORG_MAX_BYTES): + body = envelopes(name, native, org)[int(operator)] + assert len(json.dumps(body).encode()) <= MAX_BODY_BYTES + response, calls, size = await route_response(monkeypatch, name, attempt, body, operator) + assert size <= MAX_BODY_BYTES and response.status_code == 200 + assert len(calls) == 1 + stored["fields"]["future"] = "x" * MAX_BODY_BYTES + assert position in directory.begin_refusal(attaching(name), begin(stored, position)) From d6e5eec3e40ff40a9fce03cae4a596cc8e701501 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:00:13 +0000 Subject: [PATCH 085/181] Scope SCIM credentials to the configured cell Signed-off-by: zlareb1 --- docs/architecture/cells.md | 25 +++- docs/operations/sso.md | 23 ++- docs/reference/cli.md | 2 + docs/reference/configuration.md | 2 +- server/auth/scim.py | 34 ++++- server/auth/scim_routes.py | 46 ++++-- server/boot.py | 15 ++ server/cli/cell.py | 14 ++ server/cli/doctor.py | 16 ++- server/cli/doctor_cell.py | 39 +++-- server/core/cell.py | 22 ++- server/core/scim_tokens.py | 50 +++++++ server/identity/app/scim_groups.py | 13 +- server/settings/catalog.py | 2 + server/worker_main.py | 9 +- tests/unit/snapshots/cli_help.json | 13 +- tests/unit/snapshots/cli_tree.json | 137 ++++++++++++++++++ tests/unit/test_cell.py | 2 +- tests/unit/test_scim_cell_authorization.py | 147 +++++++++++++++++++ tests/unit/test_scim_cell_cli.py | 61 ++++++++ tests/unit/test_scim_cell_doctor.py | 102 +++++++++++++ tests/unit/test_scim_cell_lifecycle.py | 159 +++++++++++++++++++++ tests/unit/test_scim_cell_tokens.py | 104 ++++++++++++++ 23 files changed, 980 insertions(+), 57 deletions(-) create mode 100644 server/core/scim_tokens.py create mode 100644 tests/unit/test_scim_cell_authorization.py create mode 100644 tests/unit/test_scim_cell_cli.py create mode 100644 tests/unit/test_scim_cell_doctor.py create mode 100644 tests/unit/test_scim_cell_lifecycle.py create mode 100644 tests/unit/test_scim_cell_tokens.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..b5292f966c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -278,10 +278,14 @@ unique in the directory). A token that starts with `lsc_` names a cell and must without that prefix names no cell. With `LOOM_CELL` set, a SCIM token has the form `lsc__`, as `loom cell scim-token new --cell ` makes it (it runs offline): exactly three fields split on `_`, the middle one equal to `LOOM_CELL`, and a -secret of at least 32 lowercase hex characters (128 bits). - -One parser beside `scim.tokens()` returns a token's cell name or nothing, and its -callers compare that with `LOOM_CELL` and the cell's `first` row (below). +secret of at least 32 lowercase hex characters. The offline command draws 32 random +bytes and prints 64 lowercase hex characters; a configured token's length alone does +not establish its randomness. + +The pure parser and configuration policy in `server.core.scim_tokens` share the token +splitter exposed as `scim.tokens()`. Core admission and authentication use the same +policy without a second identity query or cache. Callers compare the token's name +with `LOOM_CELL` and the full immutable `first` tuple (below). `scim_settings_problem` takes both, so boot and `loom doctor` refuse through one function, and with `LOOM_CELL` set they refuse a third configured token (two allow a rotation). One function, `scim.authorized(presented)`, holds the constant-time @@ -291,6 +295,19 @@ nothing. With it set it compares against the tokens the parser admits for `LOOM_ and the `first` row; while that row is unread it raises as `AuthUnavailable` does, so the routes answer 503 with `Retry-After` and the identity provider retries. +The cell admission gate derives token policy before returning a cached readiness +verdict. A late first successful identity read cannot admit an invalid configuration. +Server and worker startup check known identity before starting writers; unknown +identity keeps the process unready while its deployment probe remains available for +prepare. Doctor grades policy from its own bounded state query, including both fields +of the first tuple. Unread identity fails the configured SCIM check. + +Store generated tokens in the operator's secret store and configure the identity +provider for that cell's host. To rotate, configure current and replacement tokens, +switch the identity provider to the replacement, verify requests, then remove the old +entry. Two configured entries are allowed per cell, including duplicate entries in +that count. Standalone deployments retain their existing configured-token behavior. + - A process refuses to boot with a SCIM token that names another cell, and `/scim/v2` answers 401 to one. - A later cell refuses a token that names no cell. The first cell, which keeps the diff --git a/docs/operations/sso.md b/docs/operations/sso.md index 43ac40cca3..4138c604db 100644 --- a/docs/operations/sso.md +++ b/docs/operations/sso.md @@ -462,12 +462,27 @@ the tokens they made are refused at once by the Loom process that took the request, and by every other process, on that replica or another, at its next revocation refresh (`LOOM_JWT_REVOCATION_CACHE_TTL_SECONDS`, 30 s by default). No process makes them a new token meanwhile: every mint reads revocations from -ClickHouse. Set a long random token and restart: +ClickHouse. On a standalone deployment, set a long random token and restart: ```bash LOOM_SCIM_TOKEN=$(openssl rand -hex 32) ``` +With `LOOM_CELL` set, generate a token for that exact cell name: + +```bash +loom cell scim-token new --cell +``` + +This runs offline and prints only the token. Keep it in the operator's secret +store and set it as `LOOM_SCIM_TOKEN` and the provider application's bearer for +that cell's host. A cell accepts only its own named tokens. Only the observed +first cell also accepts existing unnamed tokens, so its rollout needs no forced +rotation. A cell allows at most two configured entries, including duplicates. +An unread first-cell identity answers 503 with `Retry-After`; `loom doctor` +fails its SCIM check until it can read the identity. See [cell token +policy](../architecture/cells.md#scim-and-sign-in-across-cells). + Empty (the default) leaves SCIM off and its routes unmounted. Loom refuses to start with a token shorter than 32 characters, or with SCIM set while sign-in is not SSO (`LOOM_OAUTH_LOGIN_PROVIDER=oidc`) or auth is off: a deactivation @@ -479,9 +494,13 @@ comma-separated, and roll the replicas; set the new one at the provider; then remove the old one and roll again: ```bash -LOOM_SCIM_TOKEN="$OLD_TOKEN,$(openssl rand -hex 32)" +LOOM_SCIM_TOKEN="$OLD_TOKEN,$REPLACEMENT_TOKEN" ``` +Generate the replacement with the same command used for that deployment's +initial token. Both entries on a cell must name that cell, except for unnamed +tokens retained by its observed first cell. + Loom keys a SCIM user on an email: of the request's `emails` entries with an address, the one marked primary, else the first work one, else the first; or its `userName` when it sends none (Loom logs a warning when a user is created diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..4d017cb034 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -325,6 +325,8 @@ python scripts/codegen/docs_reference.py | `loom router serve` | Serve the placement, lookup and operator APIs; route other requests to cells. | `--host`, `--port`, `--log-level` | | `loom router migrate` | Apply the directory's pending migrations. | | | `loom cell` | Operate a deployment's cells through the router's directory. | | +| `loom cell scim-token` | Generate a cell's SCIM token offline. | | +| `loom cell scim-token new` | Print a random token for the named cell. | `--cell` | | `loom cell credential` | Make a router credential, or rotate a cell's. | ``, `--cell`, `--next`, `--finish`, `--revert`, `--operator`, `--router` | | `loom cell limit` | Close, reopen or bound a cell to new orgs. | `--cell`, `--closed`, `--open`, `--placements-max`, `--router` | | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d9f6f6eea8..1712273dda 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -85,7 +85,7 @@ Authentication, tokens and identity providers. | `LOOM_OIDC_JIT_ORG` | `str` | `''` | | Org that JIT-provisioned accounts are created in; required when `oidc_jit_provision` is on. | | `LOOM_OIDC_JIT_EMAIL_DOMAINS` | `str` | `''` | | Comma-separated email domains JIT provisioning admits; empty admits any verified email. | | `LOOM_IDENTITY_PROVIDER_TIMEOUT_S` | `float` | `10.0` | live | Seconds one call to an identity provider may take: OIDC discovery, the token and revocation endpoints, its signing keys, a machine issuer's discovery, and an end-user issuer's JWKS fetch. | -| `LOOM_SCIM_TOKEN` | `str` | *(unset)* | | Bearer tokens (comma-separated, any one accepted) an identity provider presents to deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. | +| `LOOM_SCIM_TOKEN` | `str` | *(unset)* | | Bearer tokens (comma-separated, any one accepted) an identity provider presents to deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. With LOOM_CELL set, at most two entries, named for that cell; only the observed first cell also accepts unnamed tokens. Generate one with `loom cell scim-token new --cell NAME`. | | `LOOM_JWT_SECRET` | `str` | *(set)* | | The root secret, and the one secret a deployment carries: it signs tokens as kid `default` and seals every stored credential and every signing key in loom.signing_keys (`loom signing-key rotate` adds those). Required when auth is on. The default is the checked-in DEV secret anyone can mint with: a deployment with auth on refuses to start with it, except the `make dev` stack (`loom doctor` warns everywhere else). A non-loopback bind refuses an HMAC secret under 32 bytes or an RSA key under 2048 bits. | | `LOOM_JWT_SECRET_PREVIOUS` | `str` | *(unset)* | | The root secret LOOM_JWT_SECRET replaced, while a rotation is under way: credentials and signing keys sealed under it still open (tokens it signed as kid `default` do not verify: a leaked root mints nothing). `loom secrets reseal` rewrites the stored ones under the new root; then remove this. "" = none. | | `LOOM_JWT_KEYRING_REFRESH_SECONDS` | `float` | `30.0` | | Seconds between re-reads of loom.signing_keys in every process (at least 1); `loom signing-key rotate` waits out two of the slowest process's. | diff --git a/server/auth/scim.py b/server/auth/scim.py index a5d1c6791b..c99db5f447 100644 --- a/server/auth/scim.py +++ b/server/auth/scim.py @@ -27,10 +27,11 @@ from typing import Any from server.auth import login, revocation, service_accounts -from server.core import leases +from server.core import leases, scim_tokens from server.core.ch import get_client, own_write_reads from server.core.datetimes import as_aware_utc from server.core.grant_sql import LIVE_GRANT +from server.core.scim_tokens import tokens as tokens from server.core.versions import next_version from server.observability import audit from server.settings import settings @@ -73,9 +74,34 @@ class Busy(Exception): """Another request is changing this user; retry shortly.""" -def tokens(raw: str) -> list[str]: - """The SCIM bearer tokens a `LOOM_SCIM_TOKEN` value lists; none means SCIM is off.""" - return [t.strip() for t in raw.split(",") if t.strip()] +def authorized(presented: str) -> bool: + """Compare all configured candidates after the cell's token policy admits them.""" + from server.auth.jwt import AuthUnavailable, TokenError + from server.core import cell + + configured = tokens(settings.scim_token) + if not configured: + return False + named = None + if settings.cell: + try: + named = scim_tokens.token_cell(presented) + except ValueError: + raise TokenError("invalid SCIM token", reason="invalid") from None + if named is not None and named != settings.cell: + raise TokenError("SCIM token names another cell", reason="invalid") + problem = scim_tokens.cell_policy_problem( + settings.scim_token, settings.cell, cell.first_identity() + ) + if problem is None: + raise AuthUnavailable("the cell's SCIM identity is not read yet; retry") + if problem: + raise AuthUnavailable("the cell's SCIM token configuration is not valid") + matches = [hmac.compare_digest(presented.encode(), token.encode()) for token in configured] + matched = any(matches) + if named is not None and not matched: + raise TokenError("invalid SCIM token", reason="invalid") + return matched def normalized(address: str) -> str: diff --git a/server/auth/scim_routes.py b/server/auth/scim_routes.py index 25360d85ac..5295f6ce06 100644 --- a/server/auth/scim_routes.py +++ b/server/auth/scim_routes.py @@ -11,7 +11,6 @@ from __future__ import annotations -import hmac import json import logging import re @@ -27,7 +26,7 @@ from server.auth import scim from server.auth.bearer import bearer_token -from server.auth.jwt import TokenError +from server.auth.jwt import AuthUnavailable, TokenError from server.auth.public_url import public_base_url from server.core.ch import set_query_tag from server.core.datetimes import as_aware_utc @@ -42,7 +41,7 @@ _CONFIG_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:ServiceProviderConfig" MAX_RESULTS = 200 LAST_INDEX = 2**32 - 1 -# A shorter bearer token is guessable: SCIM refuses to start with one. +# Minimum configured token length; secure generation is still required. MIN_TOKEN_LENGTH = 32 log = logging.getLogger(__name__) @@ -72,10 +71,14 @@ async def _authorized(request: Request) -> None: if token is None: record_auth_refusal(None, transport="scim") raise ScimError(401, message) - # Every token is compared, so the time taken doesn't say which one matched. - presented = token.encode() - matches = [hmac.compare_digest(presented, t.encode()) for t in scim.tokens(settings.scim_token)] - if not any(matches): + try: + matched = scim.authorized(token) + except AuthUnavailable as e: + raise ScimError(503, "SCIM tokens cannot be checked yet; retry") from e + except TokenError as e: + record_auth_refusal(e, transport="scim") + raise ScimError(401, message) from None + if not matched: record_auth_refusal(TokenError(reason="invalid"), transport="scim") raise ScimError(401, message) @@ -761,9 +764,16 @@ async def unknown(rest: str, request: Request) -> Response: raise ScimError(404, "this server serves only Users, Groups and ServiceProviderConfig") -def scim_settings_problem(raw: str, login_provider: str, auth_required: bool) -> str: +def scim_settings_problem( + raw: str, + login_provider: str, + auth_required: bool, + *, + cell_name: str = "", + first: tuple[bool, str] | None = None, +) -> str: """Why SCIM, with the `LOOM_SCIM_TOKEN` value `raw`, cannot start, or '' - (SCIM off is fine): a guessable token, or sign-in that is not SSO or not + (SCIM off is fine): a short token, or sign-in that is not SSO or not required, where a deactivation would answer 200 and refuse nobody.""" tokens = scim.tokens(raw) for token in tokens: @@ -779,12 +789,24 @@ def scim_settings_problem(raw: str, login_provider: str, auth_required: bool) -> ) if tokens and not auth_required: return "LOOM_SCIM_TOKEN is set but auth is off: no sign-in is checked, so none is refused" - return "" + from server.core.scim_tokens import cell_policy_problem + + # Unknown first identity keeps boot probe-capable; the existing cell gate holds it. + return cell_policy_problem(raw, cell_name, first) or "" -def require_scim_settings(raw: str, login_provider: str, auth_required: bool) -> None: +def require_scim_settings( + raw: str, + login_provider: str, + auth_required: bool, + *, + cell_name: str = "", + first: tuple[bool, str] | None = None, +) -> None: """Refuse to start with SCIM that could not deactivate anyone (run in the lifespan).""" - if problem := scim_settings_problem(raw, login_provider, auth_required): + if problem := scim_settings_problem( + raw, login_provider, auth_required, cell_name=cell_name, first=first + ): raise RuntimeError(f"refusing to start: {problem}") diff --git a/server/boot.py b/server/boot.py index 69029d5b34..3d041ae648 100644 --- a/server/boot.py +++ b/server/boot.py @@ -625,6 +625,7 @@ async def start_cell_state() -> list[asyncio.Task[None]]: if not cell.deployment_id(): cell.set_deployment_id(await pg_profile.deployment_id()) await cell.refresh() + _require_cell_scim_settings() return [cell.start_refresh_loop()] @@ -647,6 +648,7 @@ async def start_cell_gate() -> list[asyncio.Task[None]]: ) # With Postgres migrations pending (LOOM_BOOT_MIGRATE off) the state tables may not # exist yet: the process boots, gated and unready until a read lands. + _require_cell_scim_settings() loops = [cell.start_refresh_loop()] if settings.cell: from server.core import cell_router @@ -656,6 +658,19 @@ async def start_cell_gate() -> list[asyncio.Task[None]]: return loops +def _require_cell_scim_settings() -> None: + from server.auth.scim_routes import require_scim_settings + from server.core import cell + + require_scim_settings( + settings.scim_token, + settings.oauth_login_provider, + settings.auth_actually_required(), + cell_name=settings.cell, + first=cell.first_identity(), + ) + + async def ensure_default_namespace() -> None: """An auth-off deployment serves a single configured namespace: give it a loom.namespaces row now (and warn about data stranded in others) rather than on the diff --git a/server/cli/cell.py b/server/cli/cell.py index 9cdec5c894..6e9a54d167 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -62,6 +62,14 @@ def new_credential(prefix: str = "lcr_") -> tuple[str, str]: return credential, hashlib.sha256(credential.encode()).hexdigest() +def _cmd_scim_token(args: argparse.Namespace) -> int: + if not NAME.fullmatch(args.cell): + print("loom cell scim-token: cell must match [a-z0-9-]{1,32}", file=sys.stderr) + return 2 + print(f"lsc_{args.cell}_{secrets.token_hex(32)}") + return 0 + + def parse_router(args: argparse.Namespace) -> httpx.URL | str: """The router's URL, or why it is not one.""" url = args.router or settings.cell_router_url @@ -422,6 +430,12 @@ def _add_cell_parser(sub: Any) -> None: p = sub.add_parser("cell", help="Operate a deployment's cells through the router's directory.") csub = p.add_subparsers(dest="cell_cmd", required=True) + scim = csub.add_parser("scim-token", help="Generate a cell's SCIM token offline.") + scim_sub = scim.add_subparsers(dest="action", required=True) + scim_new = scim_sub.add_parser("new", help="Print a random token for the named cell.") + scim_new.add_argument("--cell", required=True, help="The cell's exact LOOM_CELL name") + scim_new.set_defaults(func=_cmd_scim_token) + cred = csub.add_parser("credential", help="Make a router credential, or rotate a cell's.") cred.add_argument("action", nargs="?", choices=["new"], help="new: make one offline") cred.add_argument("--cell", help="The cell whose credential to rotate") diff --git a/server/cli/doctor.py b/server/cli/doctor.py index d1295ffb32..d2f22cc261 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -258,17 +258,25 @@ def _doctor_keyring_check(settings: Any) -> tuple[str, str, str]: _LOOPBACK = ("127.0.0.1", "::1") -def _doctor_scim_token_check(settings: Any) -> tuple[str, str, str]: +def _doctor_scim_token_check( + settings: Any, first: tuple[bool, str] | None = None +) -> tuple[str, str, str]: """The `scim_token` line: FAIL when the server would refuse to start with it.""" from server.auth.scim import tokens from server.auth.scim_routes import scim_settings_problem problem = scim_settings_problem( - settings.scim_token, settings.oauth_login_provider, settings.auth_actually_required() + settings.scim_token, + settings.oauth_login_provider, + settings.auth_actually_required(), + cell_name=settings.cell, + first=first, ) if problem: return ("scim_token", FAIL, problem) count = len(tokens(settings.scim_token)) + if settings.cell and count and first is None: + return ("scim_token", FAIL, "cell first identity not read; SCIM token policy not checked") return ("scim_token", PASS, f"{count} token(s) set" if count else "SCIM off") @@ -640,7 +648,7 @@ def _run_live_checks() -> list[tuple[str, str, str]]: else: jwt_grade, jwt_detail = PASS, "set" checks.append(("jwt_secret", jwt_grade, jwt_detail)) - if settings_ok: + if settings_ok and (not settings.cell or not ch_host): checks.append(_doctor_scim_token_check(settings)) # 4. ClickHouse reachability. HTTP probe avoids spinning up asyncio. @@ -649,6 +657,8 @@ def _run_live_checks() -> list[tuple[str, str, str]]: else: ch_ok, ch_detail = _doctor_ch_reachable(settings, ch_host) checks.append(("clickhouse_reachable", _grade(ch_ok), ch_detail)) + if settings_ok and settings.cell and ch_host and not ch_ok: + checks.append(_doctor_scim_token_check(settings)) # 4a. The checks boot runs before it migrates: version, database, grants. checks.append( diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index f2d6f80367..8e2ebd10ca 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -5,8 +5,8 @@ A refusal (ClickHouse access denied, no LOOM_PG_DSN, another recorded authority or a Postgres of another deployment) fails the doctor's exit and reads no cell state; -any other error is one "not read" line. None of the rest fails a doctor run used as a -deployment gate: a defect is named, graded WARN.""" +any other error is one "not read" line. Configured SCIM policy needs a readable first +identity and fails the run when invalid or unread. Marker and trigger defects are WARN.""" from __future__ import annotations @@ -61,17 +61,17 @@ async def _bounded(read: Any) -> Any: raise _Refused(str(e)) from None -async def _marked_session() -> tuple[str, bool, list[Any]]: - """The state, whether this process's sessions keep the marker after RESET ALL, and +async def _marked_session() -> tuple[str, tuple[bool, str] | None, bool, list[Any]]: + """The state and first tuple, whether sessions keep the marker after RESET ALL, and the triggers' defects, on one session opened as the pool's are.""" from server.core import cell, cell_triggers, pg async with pg.dedicated(statement_timeout_ms=_STATEMENT_MS) as conn: - state = await cell.read_state(conn) + state, first = await cell.read_state_and_first(conn) await conn.execute("RESET ALL") # a startup value survives it, a SET does not marked = await conn.fetchval("SELECT current_setting($1, true)", pg.TENANCY_MARKER) defects = await cell_triggers.missing(conn) - return state, marked == "on", defects + return state, first, marked == "on", defects async def _unmarked_session() -> bool: @@ -150,16 +150,31 @@ def _grades( async def _checks(ch: Any) -> list[tuple[str, str, str]]: + from server.cli.doctor import _doctor_scim_token_check + from server.settings import settings + try: if before := await asyncio.wait_for(_deployment_check(ch), timeout=_SESSION_S): - return [("cell_state", PASS, f"cell checks not run: {before}")] - state, marked, defects = await _bounded(_marked_session()) + return [("cell_state", PASS, f"cell checks not run: {before}")] + _unread_scim() + state, first, marked, defects = await _bounded(_marked_session()) marks_all = await _bounded(_unmarked_session()) except _Refused as e: - return [("cell_deployment", FAIL, f"deployment check refused: {e}")] + return [("cell_deployment", FAIL, f"deployment check refused: {e}")] + _unread_scim() except Exception as e: # noqa: BLE001 — an unreadable cell is named, not a failed run - return [("cell_state", WARN, f"not read: {_reason(e)}")] - return _grades(state, marked, marks_all, defects) + return [("cell_state", WARN, f"not read: {_reason(e)}")] + _unread_scim() + return _grades(state, marked, marks_all, defects) + [_doctor_scim_token_check(settings, first)] + + +def _unread_scim() -> list[tuple[str, str, str]]: + from server.cli.doctor import _doctor_scim_token_check + from server.core.scim_tokens import tokens + from server.settings import settings + + return ( + [_doctor_scim_token_check(settings)] + if settings.cell and tokens(settings.scim_token) + else [] + ) def _reason(e: BaseException) -> str: @@ -182,7 +197,7 @@ async def _go() -> list[tuple[str, str, str]]: ) return await _checks(ch) except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named - return [("cell_state", WARN, f"not read: {_reason(e)}")] + return [("cell_state", WARN, f"not read: {_reason(e)}")] + _unread_scim() finally: await disconnect(ch) diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..69c3565058 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -44,7 +44,7 @@ from typing import Any from urllib.parse import urlsplit -from server.core import pg, readiness +from server.core import pg, readiness, scim_tokens from server.core.errors import caller_text from server.settings import settings from server.settings.catalog import is_http_url @@ -298,6 +298,11 @@ def is_first() -> bool | None: return None if _view.first is None else _view.first[0] +def first_identity() -> tuple[bool, str] | None: + """The immutable first tuple in the existing state view, None before it is known.""" + return _view.first + + def rows_read() -> bool: """Whether a read of the state rows has landed in this process.""" return _view.read_at is not None @@ -350,6 +355,8 @@ def gate_reason() -> str: # and a gated one stays gated (docs/architecture/cells.md, "Invariants"). return _unset_reason(_view) view, memo = _view, _memo + if problem := scim_tokens.cell_policy_problem(settings.scim_token, settings.cell, view.first): + return problem # Checked on every call, before the verdict kept for this read: time moves on. A # failed read trusts the last rows as long as a revocation snapshot is trusted. max_age = float(settings.jwt_revocation_max_stale_seconds) @@ -496,11 +503,11 @@ def readiness_check() -> readiness.CheckResult: async def read_state(conn: Any) -> str: """The cell state on `conn` (`state`); raises when it cannot read.""" - return (await _read_named_state(conn))[0] + return (await read_state_and_first(conn))[0] -async def _read_named_state(conn: Any) -> tuple[str, str]: - """The cell state on `conn` and the cell its first row names ('' without one).""" +async def read_state_and_first(conn: Any) -> tuple[str, tuple[bool, str] | None]: + """The state and full immutable first identity from the same statement.""" row = await conn.fetchrow(_STATE) first = None if row["is_first"] is None else (bool(row["is_first"]), str(row["cell"])) found = state( @@ -509,14 +516,14 @@ async def _read_named_state(conn: Any) -> tuple[str, str]: has_own=bool(row["has_own"]), has_tenancy=bool(row["has_tenancy"]), ) - return found, first[1] if first else "" + return found, first _STATE_TABLES = ("loom.cell_first", "loom.tenancy_mode", "loom.cell_self", "loom.tenancy") async def read_state_if_present(conn: Any) -> tuple[str, str]: - """`_read_named_state`, with every state table absent read as `open`: a schema that + """The state and first name, with every state table absent read as `open`: a schema that predates them cannot be a cell. Judged by `to_regclass`, never by matching a privilege error, so a role missing a grant still raises; some tables absent is `partial`.""" @@ -527,7 +534,8 @@ async def read_state_if_present(conn: Any) -> tuple[str, str]: return "open", "" if present < len(_STATE_TABLES): return "partial", "" - return await _read_named_state(conn) + found, first = await read_state_and_first(conn) + return found, first[1] if first else "" async def refresh() -> bool: diff --git a/server/core/scim_tokens.py b/server/core/scim_tokens.py new file mode 100644 index 0000000000..d93d864024 --- /dev/null +++ b/server/core/scim_tokens.py @@ -0,0 +1,50 @@ +"""Pure SCIM token syntax and cell configuration policy.""" + +from __future__ import annotations + +import re + +_NAMED = re.compile(r"lsc_([a-z0-9-]{1,32})_[0-9a-f]{32,}") + + +def tokens(raw: str) -> list[str]: + """Configured bearer entries; duplicates count and blank entries do not.""" + return [token.strip() for token in raw.split(",") if token.strip()] + + +def token_cell(token: str) -> str | None: + """The named cell, None for an unnamed token; malformed reserved names refuse.""" + if not token.startswith("lsc_"): + return None + match = _NAMED.fullmatch(token) + if match is None: + raise ValueError("a named SCIM token has invalid syntax") + return match[1] + + +def cell_policy_problem(raw: str, name: str, first: tuple[bool, str] | None) -> str | None: + """A refusal, '' for valid policy, or None while first identity is unread.""" + if not name: + return "" + configured = tokens(raw) + if not configured: + return "" + if len(configured) > 2: + return "a cell accepts at most two configured SCIM tokens" + unnamed = False + for token in configured: + try: + named = token_cell(token) + except ValueError: + return "a configured named SCIM token has invalid syntax" + if named is None: + unnamed = True + elif named != name: + return "a configured SCIM token names another cell" + if first is None: + return None + if first[1] != name: + return "the SCIM first identity names another cell" + if unnamed and not first[0]: + return "an unnamed SCIM token is accepted only on the first cell" + return "" diff --git a/server/identity/app/scim_groups.py b/server/identity/app/scim_groups.py index 6ee66d219a..cfe69eaa88 100644 --- a/server/identity/app/scim_groups.py +++ b/server/identity/app/scim_groups.py @@ -15,7 +15,6 @@ from __future__ import annotations -import hmac import re import time import uuid @@ -49,7 +48,6 @@ from server.observability import audit from server.observability.metrics import record_auth_refusal from server.operations.base import Role -from server.settings import settings GROUP_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:Group" # The path Entra and Okta send to remove one member: members[value eq ""]. @@ -89,9 +87,14 @@ async def _caller(request: Request) -> _Caller: if token is None: record_auth_refusal(None, transport="scim") raise ScimError(401, message) - presented = token.encode() - if any([hmac.compare_digest(presented, t.encode()) for t in scim.tokens(settings.scim_token)]): - return _Caller(org="", actor="scim") + try: + if scim.authorized(token): + return _Caller(org="", actor="scim") + except AuthUnavailable as e: + raise ScimError(503, "SCIM tokens cannot be checked yet; retry") from e + except TokenError as e: + record_auth_refusal(e, transport="scim") + raise ScimError(401, message) from None try: auth: AuthInfo = verify_token(token) except AuthUnavailable as e: # the revocation list or keyring is not loaded yet diff --git a/server/settings/catalog.py b/server/settings/catalog.py index b98c41fbbb..f468a142c0 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -275,6 +275,8 @@ def _check_default_namespace(self) -> AuthSettings: identity_provider_timeout_s: Annotated[float, LIVE] = Field(default=10.0, gt=0) # Bearer tokens (comma-separated, any one accepted) an identity provider presents to # deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. + # With LOOM_CELL set, at most two entries, named for that cell; only the observed first + # cell also accepts unnamed tokens. Generate one with `loom cell scim-token new --cell NAME`. scim_token: str = "" @property diff --git a/server/worker_main.py b/server/worker_main.py index 6854be4082..1fa764cad1 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -103,23 +103,24 @@ async def _serve(fd: int, alive_fd: int) -> None: ch = get_client() await ch.connect() await boot.open_storage_authority() + services = [] try: - services = await _start_services(ch) if settings.cell: # The passes verify tokens and judge the gate as the parent does: the child # reads the cell's rows itself. services += await boot.start_cell_state() + services += await _start_services(ch) supervisor = get_supervisor() supervisor.start_all() await stop.wait() log.info("workers child %s stopping", os.getpid()) embedder_calibration.stop_refresh_loop() - for task in services: - task.cancel() - await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() finally: + for task in services: + task.cancel() + await asyncio.gather(*services, return_exceptions=True) beat.cancel() await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S) await boot.close_storage_authority() diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..ec54d5b8e7 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -434,10 +434,19 @@ "options: -h, --help show this help message and exit" ], "loom cell": [ - "usage: loom cell [-h] {credential,limit,which,remove,prepare} ...", - "positional arguments: {credential,limit,which,remove,prepare} credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", + "usage: loom cell [-h] {scim-token,credential,limit,which,remove,prepare} ...", + "positional arguments: {scim-token,credential,limit,which,remove,prepare} scim-token Generate a cell's SCIM token offline. credential Make a router credential, or rotate a cell's. limit Close, reopen or bound a cell to new orgs. which Print the lookup's cell and host for an org (the first cell if not placed). remove Remove the directory row of a cell attaching with no placement. prepare Register this cell in the router's directory and write its rows.", "options: -h, --help show this help message and exit" ], + "loom cell scim-token": [ + "usage: loom cell scim-token [-h] {new} ...", + "positional arguments: {new} new Print a random token for the named cell.", + "options: -h, --help show this help message and exit" + ], + "loom cell scim-token new": [ + "usage: loom cell scim-token new [-h] --cell CELL", + "options: -h, --help show this help message and exit --cell CELL The cell's exact LOOM_CELL name" + ], "loom cell credential": [ "usage: loom cell credential [-h] [--cell CELL] [--next SHA256] [--finish] [--revert] [--operator] [--router ROUTER] [{new}]", "positional arguments: {new} new: make one offline", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..2c6bc79e0b 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -12318,6 +12318,137 @@ "help": null, "metavar": null, "subcommands": { + "scim-token": { + "prog": "loom cell scim-token", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": {}, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [ + "action" + ] + }, + { + "title": "options", + "description": null, + "dests": [ + "help" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [], + "dest": "action", + "action": "_SubParsersAction", + "default": null, + "const": null, + "choices": null, + "nargs": "A...", + "type": null, + "required": true, + "help": null, + "metavar": null, + "subcommands": { + "new": { + "prog": "loom cell scim-token new", + "usage": null, + "description": null, + "epilog": null, + "formatter_class": "argparse:HelpFormatter", + "add_help": true, + "allow_abbrev": true, + "defaults": { + "func": "server.cli.cell:_cmd_scim_token" + }, + "groups": [ + { + "title": "positional arguments", + "description": null, + "dests": [] + }, + { + "title": "options", + "description": null, + "dests": [ + "help", + "cell" + ] + } + ], + "mutually_exclusive": [], + "arguments": [ + { + "option_strings": [ + "-h", + "--help" + ], + "dest": "help", + "action": "_HelpAction", + "default": "==SUPPRESS==", + "const": null, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "show this help message and exit", + "metavar": null + }, + { + "option_strings": [ + "--cell" + ], + "dest": "cell", + "action": "_StoreAction", + "default": null, + "const": null, + "choices": null, + "nargs": null, + "type": null, + "required": true, + "help": "The cell's exact LOOM_CELL name", + "metavar": null + } + ] + } + }, + "aliases_of": { + "new": "loom cell scim-token new" + }, + "choices_help": [ + { + "dest": "new", + "help": "Print a random token for the named cell.", + "metavar": "new" + } + ] + } + ] + }, "credential": { "prog": "loom cell credential", "usage": null, @@ -12906,6 +13037,7 @@ } }, "aliases_of": { + "scim-token": "loom cell scim-token", "credential": "loom cell credential", "limit": "loom cell limit", "which": "loom cell which", @@ -12913,6 +13045,11 @@ "prepare": "loom cell prepare" }, "choices_help": [ + { + "dest": "scim-token", + "help": "Generate a cell's SCIM token offline.", + "metavar": "scim-token" + }, { "dest": "credential", "help": "Make a router credential, or rotate a cell's.", diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index dcd2536ac4..7dd97bc5dd 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -579,7 +579,7 @@ async def _beat(_fd) -> None: monkeypatch.setattr(ch_mod, "get_client", lambda: _Client()) with pytest.raises(_Done): asyncio.run(worker_main._serve(0, 0)) - assert order == (["services", "cell", "start_all"] if in_a_cell else ["services", "start_all"]) + assert order == (["cell", "services", "start_all"] if in_a_cell else ["services", "start_all"]) def test_the_stale_bound_leaves_room_for_three_cell_reads() -> None: diff --git a/tests/unit/test_scim_cell_authorization.py b/tests/unit/test_scim_cell_authorization.py new file mode 100644 index 0000000000..c87ca87a75 --- /dev/null +++ b/tests/unit/test_scim_cell_authorization.py @@ -0,0 +1,147 @@ +"""Both SCIM route families use the same local cell identity and token policy.""" + +from __future__ import annotations + +import functools +import hmac + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from server.auth import scim, scim_routes +from server.auth.jwt import AuthUnavailable, mint_token, verify_token +from server.core import cell +from server.identity.app import scim_groups +from server.settings import settings +from tests.unit.test_scim_groups import store as store + +OWN = "lsc_first_" + "a" * 64 +NEXT = "lsc_first_" + "b" * 64 +FOREIGN = "lsc_another_" + "c" * 64 +LEGACY = "legacy-token-" + "x" * 40 +JWT_SECRET = "unit-cell-token-secret-" * 3 + + +@pytest.fixture +def identity(monkeypatch): + first = [True, "first"] + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", OWN) + monkeypatch.setattr( + cell, + "first_identity", + lambda: None if first[0] is None else tuple(first), + raising=False, + ) + return first + + +@pytest.fixture +def client(identity, store, monkeypatch): + monkeypatch.setattr( + scim_groups, "verify_token", functools.partial(verify_token, secret=JWT_SECRET) + ) + api = FastAPI() + scim_groups.attach(api) + scim_routes.attach(api) + return TestClient(api) + + +@pytest.mark.parametrize("presented", [OWN, NEXT, "wrong"]) +def test_every_configured_candidate_is_compared(identity, monkeypatch, presented): + monkeypatch.setattr(settings, "scim_token", f"{OWN},{NEXT}") + compare = hmac.compare_digest + seen = [] + + def compared(left, right): + seen.append((left, right)) + return compare(left, right) + + monkeypatch.setattr(scim.hmac, "compare_digest", compared) + assert scim.authorized(presented) is (presented in (OWN, NEXT)) + assert seen == [(presented.encode(), OWN.encode()), (presented.encode(), NEXT.encode())] + + +@pytest.mark.parametrize("raw", ["", OWN, FOREIGN, "lsc_malformed", f"{OWN},{NEXT},{FOREIGN}"]) +def test_no_cell_keeps_existing_token_comparison_without_reading_identity(monkeypatch, raw): + monkeypatch.setattr(settings, "cell", "") + monkeypatch.setattr(settings, "scim_token", raw) + + def forbidden(): + pytest.fail("single-deployment authorization must not read cell identity") + + monkeypatch.setattr(cell, "first_identity", forbidden, raising=False) + assert scim.authorized(FOREIGN) is (FOREIGN in scim.tokens(raw)) + + +def test_zero_tokens_do_not_consult_cell_identity(identity, monkeypatch): + monkeypatch.setattr(settings, "scim_token", "") + + def forbidden(): + pytest.fail("SCIM off must not consult cell identity") + + monkeypatch.setattr(cell, "first_identity", forbidden) + assert scim.authorized("org-admin") is False + + +def test_unknown_identity_is_availability_not_an_ordinary_nonmatch(identity): + identity[0] = None + with pytest.raises(AuthUnavailable): + scim.authorized(OWN) + + +@pytest.mark.parametrize("path", ["/scim/v2/ServiceProviderConfig", "/scim/v2/Groups"]) +@pytest.mark.parametrize( + "configured,first,presented,status", + [ + (OWN, (True, "first"), OWN, 200), + (OWN, (False, "first"), OWN, 200), + (LEGACY, (True, "first"), LEGACY, 200), + (LEGACY, (False, "first"), LEGACY, 503), + (OWN, (None, "first"), OWN, 503), + (OWN, (True, "another"), OWN, 503), + (FOREIGN, (True, "first"), FOREIGN, 401), + (f"{OWN},{NEXT},{OWN}", (True, "first"), OWN, 503), + (OWN, (True, "first"), FOREIGN, 401), + (OWN, (True, "first"), "lsc_first_malformed", 401), + ], +) +def test_real_route_families_share_cell_policy( + client, identity, monkeypatch, path, configured, first, presented, status +): + identity[:] = first + monkeypatch.setattr(settings, "scim_token", configured) + response = client.get(path, headers={"Authorization": f"Bearer {presented}"}) + assert response.status_code == status + if status == 503: + assert response.headers["Retry-After"] == "5" + if status != 200: + assert presented not in response.text + + +def test_groups_retains_its_real_org_admin_bearer_alternative(client): + token = mint_token("operator", "acme", ["*"], ["org_admin"], secret=JWT_SECRET) + response = client.get("/scim/v2/Groups", headers={"Authorization": f"Bearer {token}"}) + assert response.status_code == 200 and response.json()["totalResults"] == 0 + + +@pytest.mark.parametrize("token", [FOREIGN, "lsc_first_malformed"]) +def test_reserved_cell_token_cannot_fall_through_as_a_groups_admin(client, monkeypatch, token): + def forbidden(_token): + pytest.fail("a reserved SCIM credential must not use the org-admin alternative") + + monkeypatch.setattr(scim_groups, "verify_token", forbidden) + response = client.get("/scim/v2/Groups", headers={"Authorization": f"Bearer {token}"}) + assert response.status_code == 401 + + +def test_groups_availability_failure_cannot_use_an_admin_fallback(client, identity, monkeypatch): + identity[0] = None + + def forbidden(_token): + pytest.fail("an unread SCIM cell must not fall through unchecked") + + monkeypatch.setattr(scim_groups, "verify_token", forbidden) + response = client.get("/scim/v2/Groups", headers={"Authorization": "Bearer org-admin"}) + assert response.status_code == 503 and response.headers["Retry-After"] == "5" diff --git a/tests/unit/test_scim_cell_cli.py b/tests/unit/test_scim_cell_cli.py new file mode 100644 index 0000000000..e8d7a7b8ed --- /dev/null +++ b/tests/unit/test_scim_cell_cli.py @@ -0,0 +1,61 @@ +"""The actual command entry generates only a local named random secret.""" + +from __future__ import annotations + +import builtins +import io +import re +import socket + +import httpx +import pytest + +from server.cli import app +from server.core import pg +from server.core.ch import CHClient +from server.insights.app import provider_barrier, provider_command + + +@pytest.fixture +def offline(monkeypatch): + def forbidden(*_args, **_kwargs): + pytest.fail("offline token generation attempted a runtime dependency") + + monkeypatch.setenv("LOOM_ENV_FILE", "") + monkeypatch.setattr(CHClient, "connect", forbidden) + monkeypatch.setattr(pg, "open_pool", forbidden) + monkeypatch.setattr(provider_command, "model_command", forbidden) + monkeypatch.setattr(provider_barrier, "enroll_before_migrations", forbidden) + monkeypatch.setattr(provider_barrier, "register_process", forbidden) + monkeypatch.setattr(httpx.Client, "send", forbidden) + monkeypatch.setattr(httpx.AsyncClient, "send", forbidden) + monkeypatch.setattr(socket.socket, "connect", forbidden) + # Imports may read source/metadata. No file write belongs to this command. + opened = builtins.open + + def read_only(file, mode="r", *args, **kwargs): + if any(flag in mode for flag in "wax+"): + forbidden() + return opened(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", read_only) + monkeypatch.setattr(io, "open", read_only) + + +@pytest.mark.parametrize("name", ["a", "a-b", "z" * 32]) +def test_actual_command_is_offline_and_draws_distinct_native_random_secrets(offline, capsys, name): + outputs = [] + for _ in range(2): + assert app.main(["cell", "scim-token", "new", "--cell", name]) == 0 + captured = capsys.readouterr() + assert captured.err == "" + assert re.fullmatch(r"lsc_" + name + r"_[0-9a-f]{64}\n", captured.out) + outputs.append(captured.out) + assert outputs[0] != outputs[1] + + +@pytest.mark.parametrize("name", ["", "a_b", "A", "z" * 33, "a\n", "é"]) +def test_invalid_names_refuse_without_a_secret_or_runtime_dependency(offline, capsys, name): + assert app.main(["cell", "scim-token", "new", "--cell", name]) == 2 + captured = capsys.readouterr() + assert captured.out == "" and "cell" in captured.err diff --git a/tests/unit/test_scim_cell_doctor.py b/tests/unit/test_scim_cell_doctor.py new file mode 100644 index 0000000000..7ad6ed8065 --- /dev/null +++ b/tests/unit/test_scim_cell_doctor.py @@ -0,0 +1,102 @@ +"""Doctor grades the actual bounded identity read, including unread failures.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock + +import pytest + +from server.cli import doctor, doctor_cell +from server.core import cell, cell_triggers, pg +from server.settings import settings + +OWN = "lsc_first_" + "a" * 64 +LEGACY = "legacy-token-" + "x" * 40 + + +@pytest.fixture +def configured(monkeypatch): + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", OWN) + monkeypatch.setattr(settings, "oauth_login_provider", "oidc") + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(settings, "identity_provider", "gateway") + monkeypatch.setattr(doctor_cell, "_deployment_check", AsyncMock(return_value="")) + monkeypatch.setattr(doctor_cell, "_unmarked_session", AsyncMock(return_value=False)) + # The global view is deliberately unrelated; doctor must use its own bounded read. + monkeypatch.setattr(cell, "_view", cell._View(first=(True, "another"))) + + +@pytest.mark.parametrize( + "first,raw,grade", + [ + ((True, "first"), LEGACY, "PASS"), + ((False, "first"), OWN, "PASS"), + ((False, "first"), LEGACY, "FAIL"), + ((True, "another"), OWN, "FAIL"), + (None, OWN, "FAIL"), + (None, "", "PASS"), + ((True, "first"), ",".join([OWN] * 3), "FAIL"), + ], +) +async def test_doctor_uses_full_first_tuple_from_its_existing_single_read( + configured, monkeypatch, first, raw, grade +): + monkeypatch.setattr(settings, "scim_token", raw) + reads = [] + + class Conn: + async def fetchrow(self, sql): + reads.append(sql) + return { + "is_first": None if first is None else first[0], + "cell": None if first is None else first[1], + "mode": "enforced", + "has_own": True, + "has_tenancy": True, + } + + async def execute(self, _sql): + pass + + async def fetchval(self, _sql, *_args): + return "on" + + @asynccontextmanager + async def dedicated(**_kwargs): + yield Conn() + + monkeypatch.setattr(pg, "dedicated", dedicated) + monkeypatch.setattr(cell_triggers, "missing", AsyncMock(return_value=[])) + rows = await doctor_cell._checks(object()) + result = next((r for r in rows if r[0] == "scim_token"), None) + assert result is not None and result[1] == grade + assert len(reads) == 1 + assert raw not in result[2] if raw else True + + +@pytest.mark.parametrize("failure", ["before-boot", "deployment", "state"]) +async def test_unread_doctor_identity_never_becomes_a_passing_policy( + configured, monkeypatch, failure +): + if failure == "before-boot": + monkeypatch.setattr( + doctor_cell, "_deployment_check", AsyncMock(return_value="before the first boot") + ) + elif failure == "deployment": + monkeypatch.setattr( + doctor_cell, "_deployment_check", AsyncMock(side_effect=doctor_cell._Refused("refused")) + ) + monkeypatch.setattr( + doctor_cell, "_marked_session", AsyncMock(side_effect=OSError("state unreadable")) + ) + rows = await doctor_cell._checks(object()) + assert any(name == "scim_token" and grade == "FAIL" for name, grade, _ in rows) + + +@pytest.mark.parametrize("raw", [OWN, LEGACY, ""]) +def test_standalone_doctor_keeps_generic_no_store_policy(configured, monkeypatch, raw): + monkeypatch.setattr(settings, "cell", "") + monkeypatch.setattr(settings, "scim_token", raw) + assert doctor._doctor_scim_token_check(settings)[1] == "PASS" diff --git a/tests/unit/test_scim_cell_lifecycle.py b/tests/unit/test_scim_cell_lifecycle.py new file mode 100644 index 0000000000..8ac922fd11 --- /dev/null +++ b/tests/unit/test_scim_cell_lifecycle.py @@ -0,0 +1,159 @@ +"""Token policy is derived before cached admission and before startup writers.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from server import boot +from server.core import cell, cell_router, pg +from server.settings import settings +from tests.unit.clock_fake import ManualClock + +OWN = "lsc_first_" + "a" * 64 +LEGACY = "legacy-token-" + "x" * 40 +INVALID = [LEGACY, "lsc_foreign_" + "b" * 64, "lsc_first_bad", ",".join([OWN] * 3)] + + +@pytest.fixture +def lifecycle(monkeypatch): + clock = ManualClock() + monkeypatch.setattr(cell.time, "monotonic", clock.monotonic) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", OWN) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(settings, "oauth_login_provider", "oidc") + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(settings, "identity_provider", "gateway") + monkeypatch.setattr(cell, "_deployment_id", "unit-deployment") + monkeypatch.setattr(cell, "_view", cell._View()) + monkeypatch.setattr(cell, "_memo", None) + monkeypatch.setattr(cell, "_awaiting_first_read", False) + monkeypatch.setattr(cell, "_code_orgs", None) + monkeypatch.setattr(cell, "start_refresh_loop", lambda: "refresh-loop") + monkeypatch.setattr(cell_router, "start_reader", lambda: "router-loop") + monkeypatch.setattr(boot, "SKIPPED_PG_MIGRATIONS", ()) + return clock + + +def _row(first=False): + return { + "first": {"is_first": first, "cell": "first"}, + "mode": "enforced", + "own": {"current_fp": cell.as_value(cell.mine()), "next_fp": None}, + "tenancy": True, + "marked": True, + } + + +@pytest.mark.parametrize("raw", INVALID) +def test_late_first_read_cannot_open_invalid_policy(lifecycle, monkeypatch, raw): + monkeypatch.setattr(settings, "scim_token", raw) + assert cell.gate_reason() + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=_row())) + assert asyncio.run(cell.refresh()) + assert cell.gate_reason(), "a successfully refreshed later cell must remain closed" + assert not cell.readiness_check().ok + + +@pytest.mark.parametrize("raw", INVALID) +def test_policy_is_checked_before_an_accepted_view_memo(lifecycle, monkeypatch, raw): + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=_row())) + assert asyncio.run(cell.refresh()) + assert cell.gate_reason() == "" + assert cell._memo is not None + view = cell._view + monkeypatch.setattr(settings, "scim_token", raw) + assert cell._view is view + assert cell.gate_reason(), "the unchanged view memo must not bypass token policy" + + +@pytest.mark.parametrize("first,raw", [(True, LEGACY), (False, OWN)]) +def test_valid_policy_survives_bounded_read_failure(lifecycle, monkeypatch, first, raw): + monkeypatch.setattr(settings, "scim_token", raw) + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=_row(first))) + assert asyncio.run(cell.refresh()) + assert cell.gate_reason() == "" + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(side_effect=OSError("unreachable"))) + assert not asyncio.run(cell.refresh()) + assert cell.gate_reason() == "" + lifecycle.advance(settings.jwt_revocation_max_stale_seconds + 1) + assert cell.gate_reason() + + +@pytest.mark.parametrize("entry", ["start_cell_gate", "start_cell_state"]) +@pytest.mark.parametrize("raw", INVALID) +def test_known_invalid_policy_refuses_startup_before_background_tasks( + lifecycle, monkeypatch, entry, raw +): + monkeypatch.setattr(settings, "scim_token", raw) + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=_row())) + loops = [] + monkeypatch.setattr(cell, "start_refresh_loop", lambda: loops.append("started")) + with pytest.raises(RuntimeError, match="SCIM|SCIM_TOKEN"): + asyncio.run(getattr(boot, entry)()) + assert loops == [] + + +@pytest.mark.parametrize("entry", ["start_cell_gate", "start_cell_state"]) +@pytest.mark.parametrize("first,raw", [(True, LEGACY), (False, OWN)]) +def test_startup_reads_first_identity_before_judging_valid_policy( + lifecycle, monkeypatch, entry, first, raw +): + monkeypatch.setattr(settings, "scim_token", raw) + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=_row(first))) + assert cell.first_identity() is None + loops = asyncio.run(getattr(boot, entry)()) + assert loops and cell.gate_reason() == "" + + +@pytest.mark.parametrize("entry", ["start_cell_gate", "start_cell_state"]) +def test_absent_identity_keeps_probe_capable_boot_gated(lifecycle, monkeypatch, entry): + monkeypatch.setattr(settings, "scim_token", LEGACY) + monkeypatch.setattr(pg, "fetchrow_reset", AsyncMock(return_value=dict(_row(), first=None))) + loops = asyncio.run(getattr(boot, entry)()) + assert loops and cell.first_identity() is None and cell.gate_reason() + + +@pytest.mark.parametrize("failure", ["policy", "services"]) +def test_worker_checks_cell_before_services_and_drains_before_leaving( + lifecycle, monkeypatch, failure +): + from server import worker_main + from server.core import ch + from server.workers import leader + + order = [] + tasks = [] + closed = [] + + async def refuse(): + order.append("cell") + if failure == "policy": + raise RuntimeError("SCIM token policy refuses") + tasks.append(asyncio.create_task(asyncio.Event().wait())) + return list(tasks) + + async def services(_ch): + order.append("services") + raise RuntimeError("services refused") + + async def leave(*_args, **_kwargs): + closed.extend(task.cancelled() for task in tasks) + + monkeypatch.setattr(worker_main.child, "read_deadlines", lambda *_a: None) + monkeypatch.setattr(worker_main.child, "exit_past_deadline", lambda *_a: None) + monkeypatch.setattr(worker_main.child, "beat", AsyncMock()) + monkeypatch.setattr(leader, "_CURRENT", leader.current()) + monkeypatch.setattr(ch, "get_client", lambda: type("Client", (), {"connect": AsyncMock()})()) + monkeypatch.setattr(boot, "open_storage_authority", AsyncMock()) + monkeypatch.setattr(boot, "close_storage_authority", AsyncMock()) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "start_cell_state", refuse) + monkeypatch.setattr(worker_main, "_start_services", services) + with pytest.raises(RuntimeError, match="SCIM token policy|services refused"): + asyncio.run(worker_main._serve(0, 0)) + assert order == (["cell"] if failure == "policy" else ["cell", "services"]) + assert closed == ([] if failure == "policy" else [True]) diff --git a/tests/unit/test_scim_cell_tokens.py b/tests/unit/test_scim_cell_tokens.py new file mode 100644 index 0000000000..513cf73d3b --- /dev/null +++ b/tests/unit/test_scim_cell_tokens.py @@ -0,0 +1,104 @@ +"""Cell token syntax and configuration never infer an unread first identity.""" + +from __future__ import annotations + +import importlib + +import pytest + +SECRET = "a" * 64 + + +def policy(): + return importlib.import_module("server.core.scim_tokens") + + +@pytest.mark.parametrize("name", ["a", "a-b", "x" * 32]) +@pytest.mark.parametrize("size", [32, 33, 64]) +def test_named_token_uses_the_complete_exact_grammar(name, size): + assert policy().token_cell(f"lsc_{name}_{'a' * size}") == name + + +@pytest.mark.parametrize( + "token", + [ + f"lsc__{SECRET}", + f"lsc_{'x' * 33}_{SECRET}", + f"lsc_Name_{SECRET}", + f"lsc_name_{'A' * 32}", + f"lsc_name_{'a' * 31}", + f"lsc_name_{SECRET}_tail", + f"lsc_name_{SECRET}\n", + f"lsc_name_{SECRET}\x00", + f"lsc_na_me_{SECRET}", + f"lsc_namé_{SECRET}", + ], +) +def test_reserved_prefix_must_parse_in_full(token): + with pytest.raises(ValueError): + policy().token_cell(token) + + +@pytest.mark.parametrize("token", ["old-secret", "LSC_name_secret", "eyJ.payload.signature"]) +def test_only_the_reserved_lowercase_prefix_names_a_cell(token): + assert policy().token_cell(token) is None + + +def test_splitter_preserves_duplicates_and_only_ignores_empty_entries(): + assert policy().tokens(" , first, second ,first,, ") == ["first", "second", "first"] + + +@pytest.mark.parametrize("first", [None, (True, "first"), (False, "first")]) +def test_no_cell_keeps_its_existing_configuration_policy(first): + assert policy().cell_policy_problem("old,lsc_malformed,third,fourth", "", first) == "" + + +def test_scim_off_needs_no_first_identity(): + assert policy().cell_policy_problem(" , ", "first", None) == "" + + +@pytest.mark.parametrize("raw", ["old-secret", f"lsc_first_{SECRET}"]) +def test_valid_configuration_with_unknown_first_identity_stays_undecided(raw): + assert policy().cell_policy_problem(raw, "first", None) is None + + +@pytest.mark.parametrize("first", [(True, "first"), (False, "first")]) +def test_own_named_token_is_valid_on_either_known_cell_kind(first): + assert policy().cell_policy_problem(f"lsc_first_{SECRET}", "first", first) == "" + + +def test_unnamed_tokens_require_the_exact_observed_first_tuple(): + assert policy().cell_policy_problem("old-secret", "first", (True, "first")) == "" + assert policy().cell_policy_problem("old-secret", "first", (False, "first")) + assert policy().cell_policy_problem("old-secret", "first", (True, "another")) + + +@pytest.mark.parametrize( + "raw", + [ + f"lsc_another_{SECRET}", + f"lsc_first_{'A' * 32}", + ",".join([f"lsc_first_{SECRET}"] * 3), + ], +) +@pytest.mark.parametrize("first", [None, (True, "first"), (False, "first")]) +def test_invalid_configuration_refuses_even_before_a_first_read(raw, first): + problem = policy().cell_policy_problem(raw, "first", first) + assert isinstance(problem, str) and problem + assert SECRET not in problem and "A" * 32 not in problem + + +def test_rotation_accepts_two_entries_and_counts_duplicate_entries(): + raw = f"lsc_first_{SECRET},lsc_first_{'b' * 64}" + assert policy().cell_policy_problem(raw, "first", (False, "first")) == "" + assert policy().cell_policy_problem(raw + f",lsc_first_{SECRET}", "first", None) + + +def test_named_first_configuration_cannot_use_a_differently_named_cached_row(): + assert policy().cell_policy_problem(f"lsc_first_{SECRET}", "first", (True, "another")) + + +def test_mixed_named_and_unnamed_rotation_is_only_for_the_first_cell(): + raw = f"old-secret,lsc_first_{SECRET}" + assert policy().cell_policy_problem(raw, "first", (True, "first")) == "" + assert policy().cell_policy_problem(raw, "first", (False, "first")) From c8ec1d6b012fbef4885d2356bbf0201f8f1fc502 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:15:33 +0000 Subject: [PATCH 086/181] Describe trigger creation before the attach mode commit Signed-off-by: zlareb1 --- docs/architecture/cells.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 0af383cebe..32e81ada9d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -802,9 +802,9 @@ read-only scope, lists any tenancy row the directory does not place. serves past a table whose lock it cannot take, logged with the holders of its locks, or one the role cannot alter, logged as such (the third exception under Invariants). A first cell is `recording` until attach sets `enforced` while it runs, so no boot can - have created them: attach creates them itself with the same pass - (`cell_triggers.ensure`) once it has set `enforced`, and the cell goes `serving` only - once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare + have created them: attach creates them with `cell_triggers.ensure_table`, passing + `recording_first=True`, before step 2 and again before setting `enforced`. + The cell goes `serving` only once `cell_triggers.missing` names no defect. A later cell is `enforced` from prepare and gets them at its first migrating boot after prepare, which its attach requires the same way. `loom doctor` names a disabled or altered one in any state, a missing one on an `enforced` cell, and a partitioned table with an `org` column as unsupported, since From 69e9f2dd00b8e93c008dadb0fe2d2cf1a38fce4e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:28:23 +0000 Subject: [PATCH 087/181] Exercise attach cleanup with a fenced directory snapshot Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_owned_exit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_owned_exit.py b/tests/unit/test_cell_attach_owned_exit.py index 0a8ba4d2f1..bc8bc89da5 100644 --- a/tests/unit/test_cell_attach_owned_exit.py +++ b/tests/unit/test_cell_attach_owned_exit.py @@ -19,6 +19,7 @@ def test_actual_attach_owner_keeps_first_failure(owner, secondary): from server.cli.cell_attach import Run from server.cli import cell_attach_pg as apg from server.cli import cell_control + from tests.unit.test_cell_control import snapshot owner, kind = sys.argv[1:] original = ValueError('operation failed') @@ -46,7 +47,7 @@ async def opening(owned, what): owned.conn = conn async def fail(*args): raise original cell_control.Owner._open = opening router = AsyncMock() - router.call.return_value = {'state': 'attaching'} + router.call.return_value = {**snapshot(), 'state': 'attaching'} run = Run(None, argparse.Namespace(cell='test', wait=0), router, lambda _: None) run.row = {'deployment_id': 'test'} run.mode = 'recording' From 01621af7a22909ac133eeb8a6b4812b397cce132 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:37:54 +0000 Subject: [PATCH 088/181] Check trigger gate provenance in its repository checkout Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a625ab5a27..211dfd88c9 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,6 +8,7 @@ import json import socket import uuid +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -1005,7 +1006,8 @@ def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fau def git(argv, **kwargs): calls.append(argv) assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False - assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert kwargs["capture_output"] is True + assert kwargs["cwd"] == Path(__file__).resolve().parents[2] assert "shell" not in kwargs if argv[1:] == ["rev-parse", "HEAD"]: value = "wrong" if fault == "head" else fixture.build_commit From 5d043871956df9ef0d0b622c412f1cd1b6384f37 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:37:56 +0000 Subject: [PATCH 089/181] Check trigger gate provenance in its repository checkout Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index a625ab5a27..211dfd88c9 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -8,6 +8,7 @@ import json import socket import uuid +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock @@ -1005,7 +1006,8 @@ def test_source_provenance_uses_only_fixed_bounded_readonly_git(monkeypatch, fau def git(argv, **kwargs): calls.append(argv) assert argv[0] == "git" and kwargs["timeout"] == 5 and kwargs["check"] is False - assert kwargs["capture_output"] is True and kwargs["cwd"].name == "scale-attachcli" + assert kwargs["capture_output"] is True + assert kwargs["cwd"] == Path(__file__).resolve().parents[2] assert "shell" not in kwargs if argv[1:] == ["rev-parse", "HEAD"]: value = "wrong" if fault == "head" else fixture.build_commit From 17eee335bb439623e154e77ca3f70d3cf519a10f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:40:33 +0000 Subject: [PATCH 090/181] Keep native HTTP failures through client cleanup Client exit preserves request failures and selected registration refusals. A failed required exit after a successful request still stops the command. Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 15 +- server/cli/cell_control.py | 20 +- server/cli/cell_place.py | 11 +- server/cli/cell_prepare.py | 17 +- .../unit/test_cell_native_http_first_error.py | 263 ++++++++++++++++++ .../unit/test_cell_place_http_first_error.py | 169 +++++++++++ 6 files changed, 482 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_cell_native_http_first_error.py create mode 100644 tests/unit/test_cell_place_http_first_error.py diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index a8a78279fd..8fd6e7ebee 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -90,10 +90,19 @@ async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" try: - async with httpx.AsyncClient( - timeout=_HEALTH_TIMEOUT, transport=self.transport - ) as client: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: answer = r.json() except (httpx.HTTPError, ValueError): return "" diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 1966b9e62b..730c36bd42 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import contextlib import json +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -304,16 +306,22 @@ async def request( endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) try: - async with ( - asyncio.timeout(REQUEST_S), - httpx.AsyncClient( + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( timeout=cli._OPERATOR_TIMEOUT, trust_env=url.scheme == "https", transport=transport, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) as client, - ): - response = await client.request(method, endpoint, json=body) + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) except (httpx.HTTPError, TimeoutError) as e: raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index c0cc15aeb2..214c79179f 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -4,6 +4,7 @@ import argparse import asyncio +import contextlib import json import sys from typing import Any @@ -69,13 +70,21 @@ async def _directory( router: httpx.URL, name: str, org: str, deployment: str, fingerprint: dict[str, Any] ) -> None: url = cell_cli.at(router, f"/cells/v1/operator/cells/{name}/place") - async with httpx.AsyncClient(trust_env=router.scheme == "https") as client: + manager = httpx.AsyncClient(trust_env=router.scheme == "https") + client = await manager.__aenter__() + try: answer = await client.post( url, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, json={"org": org, "deployment_id": deployment, "fingerprint": fingerprint}, timeout=httpx.Timeout(REQUEST_S, connect=5.0), ) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) posted = cell_cli.router_answer(answer, {"org": org}, cell_cli.OPERATOR_REFUSALS) if posted is None: raise PlacementRefused(cell_cli.not_the_router(answer)) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..4310b4c9bb 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -301,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -318,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -340,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..4634e7d69c --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,263 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py new file mode 100644 index 0000000000..3587eb95ba --- /dev/null +++ b/tests/unit/test_cell_place_http_first_error.py @@ -0,0 +1,169 @@ +"""Placement HTTP failures keep their outcome through client and Postgres cleanup.""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest + +from server.cli import cell_place +from tests.unit import test_cell_native_http_first_error as native +from tests.unit import test_cell_place as place_helpers + +local = place_helpers.local +invoke_place = place_helpers.invoke_place +exchange = native.exchange +observed = native.observed +assert_first = native.assert_first +assert_exit_first = native.assert_exit_first +assert_close_failure = native.assert_close_failure +FIRST_ERRORS = native.FIRST_ERRORS +EXIT_ERRORS = native.EXIT_ERRORS +_BOUNDARIES = ("placement",) +_DIRECTORY = cell_place._directory + + +async def invoke(boundary): + return await cell_place._directory( + httpx.URL("https://router.test"), "eu-1", "acme", "dep-eu-1", {} + ) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_keeps_http_first_failure_and_closes_pg_once( + local, exchange, monkeypatch, first_kind, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") + kind, result = await observed(invoke_place) + assert kind == "error" + if first_kind is asyncio.CancelledError: + # A cancelled child Task reconstructs this exception when Owner reads it. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_owned_placement_required_http_close_failure_prevents_insert( + local, exchange, monkeypatch, second_kind +): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + exchange.second = second_kind("required-exit") + kind, result = await observed(invoke_place) + assert kind == "error" + if second_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.second + assert conn.calls.count("close") == 1 + assert conn.closed and not conn.terminated and not conn.committed + assert not any("INSERT" in call for call in conn.calls) + + +async def test_owned_placement_successful_http_neighbor_commits(local, exchange, monkeypatch): + conn, _ = local + monkeypatch.setattr(cell_place, "_directory", _DIRECTORY) + monkeypatch.setattr(cell_place, "_bound_s", lambda: 0.5) + assert await invoke_place() == exchange.answer + assert conn.calls.count("close") == 1 + assert conn.closed and conn.committed and not conn.terminated From dfa83dd05308c9d8f017cb9e520960d859e2ba46 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 03:46:54 +0000 Subject: [PATCH 091/181] Preserve native control request failures through HTTP cleanup Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 15 +- server/cli/cell_control.py | 20 +- server/cli/cell_prepare.py | 17 +- .../unit/test_cell_native_http_first_error.py | 263 ++++++++++++++++++ 4 files changed, 303 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_cell_native_http_first_error.py diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index a8a78279fd..8fd6e7ebee 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -90,10 +90,19 @@ async def call(self, path: str, body: dict[str, Any] | None = None) -> dict[str, async def health(self, host: str) -> str: """The deployment id `https:///v1/health/deployment` answers ('' when none).""" try: - async with httpx.AsyncClient( - timeout=_HEALTH_TIMEOUT, transport=self.transport - ) as client: + manager = httpx.AsyncClient(timeout=_HEALTH_TIMEOUT, transport=self.transport) + client = await manager.__aenter__() + try: r = await client.get(f"https://{host}/v1/health/deployment") + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + except (httpx.HTTPError, ValueError): + return "" + # A required close failure after a successful read must reach the owner. + await manager.__aexit__(None, None, None) + try: answer = r.json() except (httpx.HTTPError, ValueError): return "" diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 1966b9e62b..730c36bd42 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import contextlib import json +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -304,16 +306,22 @@ async def request( endpoint = cli.at(url, f"/cells/v1/operator/cells/{name}" + path) try: - async with ( - asyncio.timeout(REQUEST_S), - httpx.AsyncClient( + async with asyncio.timeout(REQUEST_S): + manager = httpx.AsyncClient( timeout=cli._OPERATOR_TIMEOUT, trust_env=url.scheme == "https", transport=transport, headers={"Authorization": f"Bearer {settings.router_operator_token}"}, - ) as client, - ): - response = await client.request(method, endpoint, json=body) + ) + client = await manager.__aenter__() + try: + response = await client.request(method, endpoint, json=body) + except BaseException: # Client exit preserves the request's first failure. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) except (httpx.HTTPError, TimeoutError) as e: raise _refused(f"router reply unknown ({type(e).__name__}): stop and rerun") from None answer = cli.router_answer(response, {"cell": name}, cli.OPERATOR_REFUSALS) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 6e94ae1058..4310b4c9bb 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -26,6 +26,7 @@ import argparse import asyncio +import contextlib import hashlib import json import secrets @@ -301,9 +302,11 @@ async def _register( token = settings.router_operator_token url = cell_cli.at(router, f"/cells/v1/operator/cells/{args.cell}/register") deadline = time.monotonic() + args.wait - async with httpx.AsyncClient( + manager = httpx.AsyncClient( timeout=_register_timeout(), trust_env=router.scheme == "https", transport=transport - ) as client: + ) + client = await manager.__aenter__() + try: while True: try: r = await client.post(url, json=body, headers={"Authorization": f"Bearer {token}"}) @@ -318,7 +321,7 @@ async def _register( f"{cell_cli.not_the_router(r)}; run prepare again once the router answers" ) if r.status_code == 200: - return answer + break code = answer.get("code") if code == "BACKEND_UNAVAILABLE": raise Refused( @@ -340,6 +343,14 @@ async def _register( print(f"loom cell prepare: {answer['error']}; retrying", file=sys.stderr) await _sleep(pause) + except BaseException: # Client exit preserves the first request or refusal. + with contextlib.suppress(BaseException): + await manager.__aexit__(*sys.exc_info()) + raise + else: + await manager.__aexit__(None, None, None) + return answer + def _own_rows( rows: _CellRows, answer: dict[str, Any], every: list[dict[str, Any]] diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py new file mode 100644 index 0000000000..4634e7d69c --- /dev/null +++ b/tests/unit/test_cell_native_http_first_error.py @@ -0,0 +1,263 @@ +"""Native HTTP clients preserve the first outcome through required exit.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.settings import settings + +_CLIENT = httpx.AsyncClient +_BOUNDARIES = ("control", "health", "register") +FIRST_ERRORS = ( + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) +EXIT_ERRORS = ( + RuntimeError, + ValueError, + httpx.ReadError, + TimeoutError, + asyncio.CancelledError, + SystemExit, + KeyboardInterrupt, +) + + +@pytest.fixture +def exchange(monkeypatch): + state = SimpleNamespace( + first=None, + second=None, + phase="request", + close_phase="await", + events=[], + exit_args=None, + entered=asyncio.Event(), + block=False, + answer={"org": "acme", "cell": "eu-1", "deployment_id": "dep-eu-1"}, + status=200, + close_entered=asyncio.Event(), + close_block=False, + ) + + async def handler(request): + state.events.append("request") + state.entered.set() + if state.block: + try: + await asyncio.Future() + except asyncio.CancelledError as error: + state.first = error + raise + if state.first is not None: + raise state.first + return httpx.Response(state.status, json=state.answer) + + class Client(_CLIENT): + async def __aenter__(self): + state.events.append("enter") + if state.phase == "enter": + raise state.first + return await super().__aenter__() + + def __aexit__(self, *args): + state.exit_args = args + state.events.append("exit-call") + if state.close_phase == "construct" and state.second is not None: + raise state.second + + async def close(): + state.events.append("exit-await") + state.close_entered.set() + if state.close_block: + await asyncio.Future() + await super(Client, self).__aexit__(*args) + if state.second is not None: + raise state.second + + return close() + + def client(**kwargs): + state.events.append("construct") + if state.phase == "construct": + raise state.first + kwargs["transport"] = httpx.MockTransport(handler) + return Client(**kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + monkeypatch.setattr(settings, "router_operator_token", "inert-test-token") + return state + + +async def invoke(boundary): + url = httpx.URL("https://router.test") + if boundary == "control": + return await cell_control.request(url, "eu-1", None, "/claim", {}) + if boundary == "health": + return await cell_attach.Router(url, "eu-1", None).health("eu-1.example.test") + return await cell_prepare._register(argparse.Namespace(cell="eu-1", wait=0), url, {}, None) + + +async def observed(callback): + # Native fatal values must stay in this task rather than reach the event loop. + try: + return "value", await callback() + except BaseException as error: # noqa: BLE001 — compare original native failure objects. + return "error", error + + +def assert_first(boundary, state, outcome): + kind, result = outcome + if boundary == "health" and isinstance(state.first, (httpx.HTTPError, ValueError)): + assert (kind, result) == ("value", "") + elif (boundary == "control" and isinstance(state.first, (httpx.HTTPError, TimeoutError))) or ( + boundary == "register" + and state.phase == "request" + and isinstance(state.first, httpx.HTTPError) + ): + expected = cell_prepare.Refused if boundary == "register" else cell_control.Refused + assert kind == "error" and isinstance(result, expected) + assert type(state.first).__name__ in str(result) + else: + assert kind == "error" and result is state.first + + +def assert_exit_first(boundary, state): + first = state.exit_args[1] + if boundary == "register" and isinstance(state.first, httpx.HTTPError): + assert isinstance(first, cell_prepare.Refused) + else: + assert first is state.first + + +def assert_close_failure(boundary, state, outcome): + kind, result = outcome + if boundary == "control" and isinstance(state.second, (httpx.HTTPError, TimeoutError)): + assert kind == "error" and isinstance(result, cell_control.Refused) + else: + assert kind == "error" and result is state.second + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("second_kind", (*EXIT_ERRORS, None)) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_request_first_outcome_survives_client_exit( + exchange, boundary, first_kind, second_kind, close_phase +): + exchange.first = first_kind("request-first") + exchange.second = second_kind("exit-second") if second_kind is not None else None + exchange.close_phase = close_phase + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.events[:3] == ["construct", "enter", "request"] + assert exchange.events.count("exit-call") == 1 + assert_exit_first(boundary, exchange) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +async def test_required_client_exit_failure_cannot_report_success( + exchange, boundary, second_kind, close_phase +): + exchange.second = second_kind("required-exit") + exchange.close_phase = close_phase + assert_close_failure(boundary, exchange, await observed(lambda: invoke(boundary))) + assert exchange.exit_args == (None, None, None) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_successful_client_neighbor(exchange, boundary): + kind, result = await observed(lambda: invoke(boundary)) + assert kind == "value" + expected = ( + None + if boundary == "placement" + else ("dep-eu-1" if boundary == "health" else exchange.answer) + ) + assert result == expected + assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("phase", ("construct", "enter")) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +async def test_client_acquisition_keeps_its_first_outcome(exchange, boundary, phase, first_kind): + exchange.phase = phase + exchange.first = first_kind("acquire-first") + assert_first(boundary, exchange, await observed(lambda: invoke(boundary))) + assert "request" not in exchange.events and "exit-call" not in exchange.events + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +async def test_actual_task_cancellation_keeps_its_payload(exchange, boundary, second_kind): + exchange.block = True + exchange.second = second_kind("exit-second") + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("operator-first-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) + assert result.args == ("operator-first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("boundary", _BOUNDARIES) +async def test_repeated_cancellation_during_http_exit_keeps_first_payload(exchange, boundary): + exchange.block = True + exchange.close_block = True + task = asyncio.create_task(observed(lambda: invoke(boundary))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(exchange.close_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("second_kind", EXIT_ERRORS) +@pytest.mark.parametrize("close_phase", ("construct", "await")) +@pytest.mark.parametrize("semantic", ("malformed", "backend", "probe", "refusal")) +async def test_registration_selected_refusal_survives_exit( + monkeypatch, exchange, second_kind, close_phase, semantic +): + selected = cell_prepare.Refused("registration-first") + monkeypatch.setattr(cell_prepare, "Refused", lambda message: selected) + exchange.status = 200 if semantic == "malformed" else 503 + exchange.answer = ( + {"unexpected": True} + if semantic == "malformed" + else { + "error": "registration-first", + "code": {"backend": "BACKEND_UNAVAILABLE", "probe": "PROBE_BUSY", "refusal": "OTHER"}[ + semantic + ], + } + ) + exchange.second = second_kind("exit-second") + exchange.close_phase = close_phase + kind, result = await observed(lambda: invoke("register")) + assert kind == "error" and result is selected + assert exchange.exit_args[1] is selected From 6f61b3606c80d8622e778aa23a46adde613c412c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 21:54:26 +0000 Subject: [PATCH 092/181] Retire participants only after permanent quiescence Keep request tails, actual application tasks, workers and native commands under one permanently closing process gate. Enroll before acceptance reads and retain uncertified retired rows until exact process death can be verified. Remove participant retire --yes and the retireLostPods chart value. Pod API object state and heartbeat expiry cannot authorize retirement or pruning. Signed-off-by: zlareb1 --- deploy/helm/loom/templates/_helpers.tpl | 5 +- .../helm/loom/templates/analytics-export.yaml | 2 +- deploy/helm/loom/templates/deployment.yaml | 2 +- deploy/helm/loom/templates/hooks.yaml | 4 +- deploy/helm/loom/templates/networkpolicy.yaml | 4 - deploy/helm/loom/templates/rbac.yaml | 32 - .../helm/loom/templates/serviceaccount.yaml | 2 +- deploy/helm/loom/templates/tests/health.yaml | 2 +- deploy/helm/loom/templates/workers.yaml | 2 +- deploy/helm/loom/values.yaml | 12 +- docs/architecture/cells.md | 50 +- docs/architecture/failure-and-recovery.md | 21 +- docs/architecture/provider-reconfiguration.md | 35 +- docs/operations/backup-and-recovery.md | 10 +- docs/operations/kubernetes.md | 26 +- docs/operations/replicas.md | 24 +- docs/reference/cli.md | 2 +- docs/reference/errors.md | 4 + server/auth/tenancy.py | 4 +- server/boot.py | 3 + server/cli/_ch.py | 40 +- server/cli/analytics_export.py | 17 +- server/cli/code_run.py | 3 +- server/cli/import_clickgap.py | 5 +- server/cli/import_recall.py | 13 +- server/cli/maintenance.py | 55 +- server/cli/participant.py | 27 +- server/cli/sql_access.py | 9 +- server/core/cell.py | 6 + server/core/instances.py | 3 + server/core/provider_process.py | 76 +- server/core/retirement.py | 114 +++ server/insights/app/provider_barrier.py | 82 +- server/insights/app/provider_command.py | 54 +- .../insights/store/provider_participants.py | 85 +- server/main.py | 130 ++- server/memory/app/embedder_calibration.py | 6 +- server/observability/events.py | 3 +- server/observability/problems.py | 5 +- server/transport/model_maintenance.py | 16 +- server/transport/retirement.py | 28 + server/worker_main.py | 18 +- server/workers/supervisor.py | 6 +- .../provider_process_before_retirement.py | 173 ++++ tests/integration/test_cell_retirement.py | 103 +++ .../test_provider_coordination_live_ch.py | 14 +- tests/unit/conftest.py | 9 + tests/unit/snapshots/cli_help.json | 6 +- tests/unit/snapshots/cli_tree.json | 20 +- tests/unit/test_analytics_export.py | 5 +- tests/unit/test_cell_cli_orgs.py | 9 +- tests/unit/test_cell_retirement.py | 752 ++++++++++++++++++ tests/unit/test_provider_barrier.py | 37 +- tests/unit/test_provider_command.py | 3 + tests/unit/test_provider_host_proof.py | 128 +-- tests/unit/test_reextract.py | 4 +- tests/unit/test_revocation_web_refresh.py | 30 +- tests/unit/test_shutdown_deadline.py | 2 +- 58 files changed, 1851 insertions(+), 491 deletions(-) delete mode 100644 deploy/helm/loom/templates/rbac.yaml create mode 100644 server/core/retirement.py create mode 100644 server/transport/retirement.py create mode 100644 tests/fixtures/provider_process_before_retirement.py create mode 100644 tests/integration/test_cell_retirement.py create mode 100644 tests/unit/test_cell_retirement.py diff --git a/deploy/helm/loom/templates/_helpers.tpl b/deploy/helm/loom/templates/_helpers.tpl index d0fee03c7a..fbbbdbac19 100644 --- a/deploy/helm/loom/templates/_helpers.tpl +++ b/deploy/helm/loom/templates/_helpers.tpl @@ -564,14 +564,11 @@ initContainers: {{/* Pod spec fields the Deployments, the hook Jobs and the analytics export share; `serviceAccount` names another account than the release's. */}} -{{/* `leads`: the pod may hold the workers lease, which alone reads the -Kubernetes API (retireLostPods). */}} {{- define "loom.podCommon" -}} -{{- $leads := .leads -}} {{- $sa := .serviceAccount -}} {{- with .root -}} serviceAccountName: {{ default (include "loom.serviceAccountName" .) $sa }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken (and .Values.retireLostPods $leads) }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} # Service links would inject LOOM_SERVICE_HOST and the like into Loom's env namespace. enableServiceLinks: false {{- if not .Values.sandbox.hostUsers }} diff --git a/deploy/helm/loom/templates/analytics-export.yaml b/deploy/helm/loom/templates/analytics-export.yaml index e6b0355650..403d7f0009 100644 --- a/deploy/helm/loom/templates/analytics-export.yaml +++ b/deploy/helm/loom/templates/analytics-export.yaml @@ -81,7 +81,7 @@ spec: app.kubernetes.io/component: analytics-export spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false "serviceAccount" $name) | nindent 10 }} + {{- include "loom.podCommon" (dict "root" $ "serviceAccount" $name) | nindent 10 }} {{- include "loom.waitForDevClickhouse" $ | nindent 10 }} containers: - name: analytics-export diff --git a/deploy/helm/loom/templates/deployment.yaml b/deploy/helm/loom/templates/deployment.yaml index 3ee7d1e98c..c53b91de59 100644 --- a/deploy/helm/loom/templates/deployment.yaml +++ b/deploy/helm/loom/templates/deployment.yaml @@ -73,7 +73,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" (not .Values.workers.enabled)) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} topologySpreadConstraints: - maxSkew: {{ .Values.topologySpread.zoneSkew }} diff --git a/deploy/helm/loom/templates/hooks.yaml b/deploy/helm/loom/templates/hooks.yaml index cdd65cccc1..d55dbe9a64 100644 --- a/deploy/helm/loom/templates/hooks.yaml +++ b/deploy/helm/loom/templates/hooks.yaml @@ -65,7 +65,7 @@ spec: app.kubernetes.io/component: {{ .name }} spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" $ "leads" false) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" $) | nindent 6 }} containers: - name: {{ .name }} image: {{ include "loom.image" $ }} @@ -83,7 +83,7 @@ spec: value: {{ $v | toString | quote }} {{- end }} # Its participant proof names the pod, so a hook pod lost mid-run is - # retired like any other (provider_process.pod_gone). + # verified against the same exact process identity as every participant. - name: POD_NAME valueFrom: { fieldRef: { fieldPath: metadata.name } } - name: POD_NAMESPACE diff --git a/deploy/helm/loom/templates/networkpolicy.yaml b/deploy/helm/loom/templates/networkpolicy.yaml index 6a1ed604cb..14903323dc 100644 --- a/deploy/helm/loom/templates/networkpolicy.yaml +++ b/deploy/helm/loom/templates/networkpolicy.yaml @@ -87,10 +87,6 @@ spec: {{- include "loom.anywhereBut" . | nindent 8 }} ports: - port: 443 - {{- if .Values.retireLostPods }} - # The Kubernetes API (443 on EKS; 6443 on most other clusters). - - port: 6443 - {{- end }} {{- if and (not .Values.devClickhouse.enabled) (ne (int .Values.clickhouse.port) 443) }} - port: {{ .Values.clickhouse.port }} {{- end }} diff --git a/deploy/helm/loom/templates/rbac.yaml b/deploy/helm/loom/templates/rbac.yaml deleted file mode 100644 index 72a0ff9d90..0000000000 --- a/deploy/helm/loom/templates/rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.retireLostPods }} -{{- if not (or .Values.serviceAccount.create .Values.serviceAccount.name) }} -{{- fail "retireLostPods grants `get pods` to the release's service account: set serviceAccount.create or serviceAccount.name, not the namespace's default" }} -{{- end }} -{{- /* Only `get` on pods: the workers leader asks whether a lost participant's -pod still exists (server/core/provider_process.py `pod_gone`). */}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: [pods] - verbs: [get] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "loom.fullname" . }} - labels: - {{- include "loom.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "loom.fullname" . }} -subjects: - - kind: ServiceAccount - name: {{ include "loom.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/deploy/helm/loom/templates/serviceaccount.yaml b/deploy/helm/loom/templates/serviceaccount.yaml index 4536a11c5c..0c5c212fd8 100644 --- a/deploy/helm/loom/templates/serviceaccount.yaml +++ b/deploy/helm/loom/templates/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} -automountServiceAccountToken: {{ or .Values.serviceAccount.automountToken .Values.retireLostPods }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} {{- end }} diff --git a/deploy/helm/loom/templates/tests/health.yaml b/deploy/helm/loom/templates/tests/health.yaml index 98a37832ac..ee21e05380 100644 --- a/deploy/helm/loom/templates/tests/health.yaml +++ b/deploy/helm/loom/templates/tests/health.yaml @@ -10,7 +10,7 @@ metadata: helm.sh/hook-delete-policy: before-hook-creation spec: restartPolicy: Never - {{- include "loom.podCommon" (dict "root" . "leads" false) | nindent 2 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 2 }} containers: - name: health image: {{ include "loom.image" . }} diff --git a/deploy/helm/loom/templates/workers.yaml b/deploy/helm/loom/templates/workers.yaml index b89cce170b..710a38079c 100644 --- a/deploy/helm/loom/templates/workers.yaml +++ b/deploy/helm/loom/templates/workers.yaml @@ -42,7 +42,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- include "loom.podCommon" (dict "root" . "leads" true) | nindent 6 }} + {{- include "loom.podCommon" (dict "root" .) | nindent 6 }} {{- if .Values.topologySpread.enabled }} # Leader and standby on different nodes. topologySpreadConstraints: diff --git a/deploy/helm/loom/values.yaml b/deploy/helm/loom/values.yaml index 73c0ee9d30..5fe18416a7 100644 --- a/deploy/helm/loom/values.yaml +++ b/deploy/helm/loom/values.yaml @@ -205,15 +205,9 @@ serviceAccount: name: "" # IRSA (eks.amazonaws.com/role-arn) or other cloud identity annotations. annotations: {} - # Mount the service-account token (retireLostPods mounts it too). + # Mount the service-account token for workloads that explicitly need it. automountToken: false -# The workers leader retires the provider participant of a pod Kubernetes has -# deleted (lost with its node, evicted past its grace period), which would -# otherwise hold the next model change until `loom participant retire`. Grants -# the service account `get` on pods in this namespace and mounts its token. -retireLostPods: true - podAnnotations: {} podLabels: {} # amd64 by default: the C++ compiler tier's scip-clang ships for x86_64 only. The @@ -411,8 +405,8 @@ assistant: # Ingress to the Loom port from `ingressFrom` only. Egress to DNS, the # release's web pods, the extractor's pods (with extractor.enabled), the dev # ClickHouse (with devClickhouse.enabled), the EKS Pod Identity agent, and TCP -# 443, the ClickHouse port, postgres.port (with postgres.host) and 6443 (with -# retireLostPods) anywhere except `egressExcept` (providers, the IdP and git remotes +# 443, the ClickHouse port and postgres.port (with postgres.host) anywhere except +# `egressExcept` (providers, the IdP and git remotes # have no fixed range); a ClickHouse or Postgres inside the cluster needs an # extraEgress rule. Off, `guardMetadata` gives the Loom pods an egress-only policy # that allows every pod and every address but `egressExcept`. diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 78e1d4c689..9cd44ad6f4 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -18,6 +18,42 @@ that every transport enforces (`auth_actually_required()`, and `LOOM_MCP_AUTH_MO refuses to boot without them. [Storage authority](storage-authority.md) describes what a cell holds and how its two stores agree. +## Participant lifecycle baseline + +A participant advertising the ninth `host_proof` field `cell-retirement-v1` +enrolls before its first fresh primary acceptance read and uses permanent guarded +retirement. A serving cell heartbeat includes its `participant_id` beside the +fingerprint, allowing a replacement to be joined to the participant registry. +Workers children enroll independently and do not publish serving instance rows. +The capability identifies installed code; it does not prove that every older +process or unsafe retirement client has disappeared. + +Before online configuration finalization is supported, stop every unsafe manual +retirement and pruning writer and prevent its binary/image from restarting. +Inventory and positively terminate every accepting server, workers child, +standalone worker and native command that overlapped those writers. Include paused +boots, already retired/pruned identities and lifecycle-capable processes started +during the overlap: old maintenance code could have hidden their rows too. Only +then start replacement accepting and maintenance processes with this lifecycle. +A shared image tag, pod deletion, readiness or heartbeat expiry is insufficient. +Keep exact PID/boot/namespace/start or trusted runtime termination evidence with +the deployment's upgrade record. No CLI flag or stored acknowledgement substitutes +for that evidence, and the runtime census does not verify the external inventory. + +Census retained legacy rows while unsafe writers are stopped. Recertify a retained +identity only after exact verifiable physical death and known-zero cross-replica +queries. Keep empty or unknown proofs quarantined. If the legacy inventory is +incomplete, stop all accepting/native-command and retirement/pruning processes, +positively terminate or fence every runtime that could execute an old binary, and +prevent old restarts. Only after that physical quiescence may the operator request +an exact audited cleanup of selected unknown legacy participant identities. There +is no automatic age-based cleanup or affirmation bypass. Restart only guarded +lifecycle processes afterward. Unknown or partitioned runtimes remain blockers. + +This baseline is an operational prerequisite, not a release-cut requirement. +Configuration finalization also needs the directory's controller-fencing protocol; +the lifecycle certificate alone cannot authorize reservation release. + ## Why whole deployments A cell shares no store, key, lease or worker with another. Every guarantee a single @@ -485,11 +521,13 @@ with the same configuration, or `--finish`, first writes `next` wherever it is m It reruns attach's overlap refusals against every other cell's directory row, counting both values of each, writes the new fingerprint as `next` to the directory and then to `loom.cell_self`, so old and new processes both serve while the new configuration rolls -out. `loom cell update --finish` waits, at most `--wait` (default 10 minutes), until -every live process reports `next` and the longest request deadline has passed; when the -change turns SCIM on or changes the cell's OIDC client it then runs attach's SCIM -precondition (refusing, or `--revoke-unprovisioned`); only then does it make `next` the -only value, in `loom.cell_self` first and the directory last. A widened issuer or +out. Finalization first narrows the cell's accepted value durably while the directory +keeps both reservations. It then requires a complete post-cutoff participant census, +permanent guarded retirement or verified exact death for every captured identity, +and a serving target replacement outside that captured set. Heartbeat expiry and +request deadlines do not prove this. Only afterward may the directory release its +old reservation. Online finalization needs the [lifecycle baseline](#participant-lifecycle-baseline) +and controller fencing; SCIM-on changes remain gated on actual per-cell SCIM support. A widened issuer or pattern is reserved in the directory before any process accepts it, and a narrowed one is released only after none does. A refusal or a wait that runs out keeps both values and prints the processes not yet reporting `next`; rerunning resumes, and @@ -514,6 +552,8 @@ version and fields, and beside them keys that no fingerprint compares: they are needs no column for them (`server.core.instances.cell_row`). Their presence proves the build that reports them; there is no generation key. +- `participant_id`: this server's provider participant UUID, for joining its actual + process lifecycle to the serving instance row. A workers child publishes no instance row. - `serves`: `LOOM_SERVES_TRAFFIC`. - `marked`: whether this process's pooled Postgres sessions keep the tenancy marker (`loom.tenancy_checked`). The cell refresh reads it with the state rows, after `RESET ALL` diff --git a/docs/architecture/failure-and-recovery.md b/docs/architecture/failure-and-recovery.md index 40f8e4b57e..d54fb025b8 100644 --- a/docs/architecture/failure-and-recovery.md +++ b/docs/architecture/failure-and-recovery.md @@ -186,17 +186,16 @@ node. Nothing else counts: an unreachable process is not a stopped one. TTL is 60 s, renewed every 20 s; a contender takes it over once it expired and the holder's participant counts as stopped. `loom provider-lock clear --holder ` frees one whose holder has no participant row, which never counts as stopped. -- **Killed mid-reconfiguration.** The transition waits on a participant whose heartbeat - expired, since a paused process may resume with admitted work, until a process on - its host proves it gone (its container restarted, or its pid exited) and retires it. - The workers leader - deletes the rows of participants retired a day ago and lists, once a minute, those a - transition would wait on as `provider_participants_stale` in its `/v1/health`. Retire - one by hand with `loom participant retire --participant --yes` once its process - is confirmed gone; the command refuses a participant whose heartbeat is fresh or whose - queries still run. -- **A boot that fails after enrolling** retires its row; a crash-looping container leaves - none behind. +- **Killed mid-reconfiguration.** An expired heartbeat remains a blocker until the + exact process dies or completes guarded shutdown. A trusted local process can + verify its PID, namespace, boot and start identity, then retire it after all + cross-replica queries settle. Use `loom participant retire --participant ` on + that trusted host. Pod deletion or replacement and namespace changes alone do + not prove death. The workers leader prunes only certified permanent retirements + older than a day; unknown legacy rows remain visible. +- **A boot that fails after enrolling** remains nonretired unless all its actual + producers and writes stop. A shutdown deadline does not turn cancellation into + a retirement certificate. A process frozen past its heartbeat (a stopped container, a VM pause) holds the transition until it resumes and drains, or an operator retires it. diff --git a/docs/architecture/provider-reconfiguration.md b/docs/architecture/provider-reconfiguration.md index 71fa716069..622fb99dc4 100644 --- a/docs/architecture/provider-reconfiguration.md +++ b/docs/architecture/provider-reconfiguration.md @@ -36,21 +36,38 @@ stopped only once it retired itself (a graceful shutdown), a later process on it host proved it gone, or an operator retired it. An expired heartbeat is no proof: a paused process can hold admitted work outside ClickHouse and write it with the old configuration once it resumes, so the transition waits for it to beat again and -drain. Each row carries its host name, kernel boot, container id (Docker's, or a -Kubernetes pod's and container name), pid namespace, pid and start time; every 10 s -each process retires the participants of its own host and boot whose pid no longer -runs that process, or that ran in its own container before the container restarted -into a new pid namespace, once no query of theirs runs. That covers a killed process -whose container or worker restarts in place; two containers that share a host name -(host networking, two containers of one pod) never retire each other. An operator who confirmed its -process is gone retires it with `loom participant retire --participant --yes`, -which refuses one whose heartbeat is fresh or whose queries still run. The workers +drain. Each row carries its host name, kernel boot, container identity, pid namespace, +pid and start time. A later process on the same trusted host and pid namespace can +prove that the exact PID exited or was reused. A changed namespace, a missing pod +object, another pod UID or a terminal API phase does not prove physical termination. +`loom participant retire --participant ` performs that exact local verification +and refuses when evidence is unavailable or any cross-replica query remains. +The workers leader's `/v1/health` lists every participant a transition would wait on as `provider_participants_stale` (`heartbeat_expired` or `queries_running`). `system.processes` is read over `clusterAllReplicas(LOOM_CH_CLUSTER)`; a server without that cluster answers for itself, and any other failure of the read counts the participant as running. +Permanent retirement uses a separate process gate that never reopens. The outer +HTTP/WebSocket scope covers request-tail settlement; an inner scope covers actual +application tasks across middleware child-task boundaries, including OAuth and +control handlers. Worker passes and native command bodies use the same counter. +Shutdown closes admission, stops producers, waits for actual scope and registered +descendant completion, flushes buffers and checks carried and server-side writes. +Canceling a coroutine or exhausting the shutdown budget leaves its row nonretired. +A failed startup likewise cannot certify unfinished producers. Model-transition +`drained` is reversible and does not certify permanent retirement. + +The ninth `host_proof` field, `cell-retirement-v1`, identifies this lifecycle. +Only the guarded shutdown or verified-death writer publishes certified retirement. +The generic heartbeat writer cannot retire or reopen a retired identity; a +replacement process draws a fresh UUID. Pruning checks the latest certified row +before deleting all versions. Uncertified legacy retired rows stay visible and +cannot disappear through an age-based prune. Old reapers fail closed on the ninth +field, but old manual/pruning binaries must be stopped before the +[cells lifecycle baseline](cells.md#participant-lifecycle-baseline) is eligible. + Each bounded rebuild step holds the shared configuration Keeper key, renewed every 20 s (TTL 60 s). A contender takes over an expired key only when its holder's participant counts as stopped; an elapsed TTL alone is no proof. `loom diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 02de0a5c91..b2dfa6a12c 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -504,11 +504,11 @@ Bounded exceptions: - The events stream (budget violations) is drained on `SIGTERM`. A `SIGKILL` or OOM loses at most one flush interval of that telemetry, never audit or memory rows. -- Provider reconfiguration waits on a dead process until a process on its host proves it - gone, or, on Kubernetes, until the workers leader sees its pod gone. Elsewhere, one - whose host is gone too (a move to a new host) is retired by hand with - `loom participant retire --participant --yes` - ([provider reconfiguration](../architecture/provider-reconfiguration.md)). +- Provider reconfiguration waits on a process until guarded retirement or exact + physical-death proof with settled queries. On its trusted host, run + `loom participant retire --participant `. Pod deletion and heartbeat expiry + are insufficient. If its identity is unknown, keep the row until the + [physically quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). - Rate-limit bucket fill is per host. It refills in seconds, so a new host starts with full buckets. diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index 8eb2749050..3e31ef253e 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -189,17 +189,19 @@ graceful stop, and within the 30 s lease TTL when a pod or node dies. Every pod labelled `deploy:k8s:`, so a new leader runs the workers whose last pass is overdue rather than waiting out their interval again. -A pod that dies without stopping (node loss, eviction past the grace period) leaves its -provider participant row, which would hold the next embedding or model change from the -dashboard. With `retireLostPods` (the default) the workers leader retires it once its -heartbeat expired, no query of it runs and Kubernetes says the pod is gone: its object -deleted or replaced, or the pod `Failed` (evicted). That is a minute or two after a -deletion or an eviction, and after a node loss once Kubernetes deletes the node's pods -(five minutes of `NotReady` by default). It needs `get` on pods in the namespace (a Role -the chart creates for the release's service account) and that account's token, which only -the pods that may hold the workers lease mount. Without it, `/v1/health` names the row; -once the pod is gone, retire it with -`loom participant retire --participant --yes` ([replicas](./replicas.md#stopping-a-replica)). +A pod that dies without stopping leaves its provider participant row. A missing +or replaced pod object, a Failed/Succeeded API phase and a completed rollout do +not prove that an old process on a partitioned node stopped. Such rows remain +blockers until guarded shutdown or verified exact physical death with settled +queries. Loom requests no pod-read permission or service-account token for +participant retirement. The explicit `serviceAccount.automountToken` value stays +available for workloads that need a token for another integration. + +Use `loom participant retire --participant ` on the trusted host where its +exact PID/namespace/start identity is verifiable. If that evidence is unavailable, +use the [quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline) +after positively terminating or fencing every relevant runtime. API object +cleanup alone cannot authorize retirement. There is no migration Job. Every pod migrates at boot under a ClickHouse lease, so a Job would only repeat that step, and a pre-upgrade hook would run the new schema while @@ -388,7 +390,7 @@ NetworkPolicy whose one rule allows every pod and every address but those (none release: Helm refuses to take over an object it does not own, so the upgrade fails. - An engine that keeps address ranges off node IPs (Cilium's default) blocks host-network endpoints and the nodes' own addresses: NodeLocal DNSCache (so DNS), the EKS Pod - Identity agent, and the API server (which `retireLostPods` calls) where the engine + Identity agent, where the engine gives it an identity of its own. Turn on the engine's node matching, or set `guardMetadata: false` and allow them in your own policy. diff --git a/docs/operations/replicas.md b/docs/operations/replicas.md index f4c8729ef4..e635c0a490 100644 --- a/docs/operations/replicas.md +++ b/docs/operations/replicas.md @@ -101,14 +101,22 @@ it. ## Stopping a replica -Stop it gracefully (`docker stop`, SIGTERM): it marks its heartbeat row stopped -and retires its provider participant row. A forced kill leaves both; the -heartbeat row stops counting after three intervals. A process that restarts in -the same container retires the dead one's participant row itself. If a provider -reconfiguration waits on a dead process's row from elsewhere, retire it with the -`participant_id` its `/v1/health` reported, once you have confirmed it is gone -(the command refuses one whose heartbeat is fresh or whose queries still run): +Stop it gracefully (`docker stop`, SIGTERM). It first closes admission permanently, +stops its producers, and waits for actual request, worker and registered descendant +completion. It retires only after buffered, carried and server-side writes settle. +A canceled task that keeps executing or an exhausted shutdown budget leaves the +participant nonretired. A stopped or expired instance heartbeat is diagnostic. + +A trusted process on the same host and pid namespace can prove the exact old PID +exited or was reused, then retire it after all cross-replica queries settle: ```bash -loom participant retire --participant --yes +loom participant retire --participant ``` + +The command verifies retained boot, PID namespace and process-start identity; it +refuses remote or unknown identity evidence and live/uncountable queries. A +changed container namespace or Kubernetes API object is insufficient. An operator +assertion does not override these checks. For an inaccessible host, positively +terminate or fence the exact runtime before the documented +[quiescent lifecycle recovery](../architecture/cells.md#participant-lifecycle-baseline). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ff91f44681..3e0b9068ea 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -266,7 +266,7 @@ python scripts/codegen/docs_reference.py | `loom down` | Stop the local stack while preserving data; --delete-data performs a confirmed reset. | `--root`, `--file`, `--env-file`, `--delete-data`, `-y` | | `loom ch-setup` | Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. | `--print-sql`, `--print-analyst-sql` | | `loom participant` | The Loom processes a provider change waits for. | | -| `loom participant retire` | Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health). | `--participant`, `--yes` | +| `loom participant retire` | Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health). | `--participant` | | `loom deployment` | The configuration every replica checks itself against. | | | `loom deployment publish` | Publish this env's configuration as the deployment's, replacing its root secret or LOOM_EMBEDDING_MODEL/DIM on purpose. | `--yes`, `--check` | | `loom settings` | Every setting's value in force, and overrides of the live ones. | | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 62a4a86f46..792a3cd4ba 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -752,6 +752,10 @@ Raised in `server/router/app.py`. Category `capacity`, retryable: yes. Raised in `server/router/app.py`. Category `dependency`, retryable: yes. +#### `PROCESS_STOPPING` + +Raised in `server/transport/model_maintenance.py`, `server/transport/retirement.py`. + #### `PROJECTION_BEHIND` HTTP 503, raised as `server.memory.app.pg_admission.ProjectionBehindError`. 503 — earlier commits in this namespace are not yet in ClickHouse, so a read of ClickHouse that must see every write the namespace committed would miss them. Category `capacity`, retryable: yes for an idempotent op, no for another op once it ran. diff --git a/server/auth/tenancy.py b/server/auth/tenancy.py index 8c6b4c9437..5042cb22b5 100644 --- a/server/auth/tenancy.py +++ b/server/auth/tenancy.py @@ -262,7 +262,9 @@ async def _place_once(org: str) -> tuple[str, str]: request cancelled leaves the call running for the others.""" task = _placing.get(org) if task is None: - task = asyncio.create_task(_place(org)) + from server.core.background import create_deferred_task + + task = create_deferred_task(_place(org), name="tenancy-placement") _placing[org] = task task.add_done_callback(functools.partial(_placed, org)) return await asyncio.shield(task) diff --git a/server/boot.py b/server/boot.py index 69029d5b34..90ba996694 100644 --- a/server/boot.py +++ b/server/boot.py @@ -463,6 +463,9 @@ def _spawn_boot_loop(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Ta """A task that runs for the process's life: cancelled at shutdown, never registered as deferred, since a model transition's drain waits for those.""" task = asyncio.get_running_loop().create_task(coro, name=name) + from server.core.retirement import get_gate + + get_gate().track(task) _BOOT_TASKS.add(task) task.add_done_callback(_BOOT_TASKS.discard) return task diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 042d8d1de0..184aa1f6eb 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -295,10 +295,7 @@ async def connect_declared( from server.core.pg_profile import check_configured await check_configured(ch) - if org_free is not None and not named: - await check_org_free(org_free) - else: - await check_orgs(named) + await check_acceptance(orgs=named, org_free=org_free) except BaseException: await disconnect(ch) raise @@ -315,6 +312,38 @@ async def connect_code_plane(ch: CHClient, args: argparse.Namespace) -> None: await connect_declared(ch, orgs=() if getattr(args, "dry_run", False) else (org,)) +async def check_acceptance( + *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None +) -> None: + """Fresh post-enrollment gate check on already verified, open paired stores.""" + named = _declared(orgs, org_free) + if org_free is not None and not named: + await check_org_free(org_free) + else: + await check_orgs(named) + + +async def connect_verified(ch: CHClient) -> None: + """Verify paired stores without obtaining an acceptance view or admitting work. + + Native commands enroll next, then repeat their accepting org/cell checks. + Operator-only commands may keep using the org-free checked composition. + """ + from server.core.pg_profile import check_configured + + if not ch.connected: + await ch.connect() + try: + await check_configured(ch) + except BaseException: + await disconnect(ch) + raise + from server import boot + from server.core.background import register_buffer_drainer + + register_buffer_drainer("boot-writes", boot.drain_writes) + + async def connect_checked( ch: CHClient, *, orgs: Iterable[str | None] | None = None, org_free: OrgFree | None = None ) -> None: @@ -354,9 +383,10 @@ def run_connected[T]( async def _go() -> T: ch = get_client() - await connect_checked(ch, orgs=named, org_free=org_free) + await connect_verified(ch) try: async with model_command(): + await check_acceptance(orgs=named, org_free=org_free) return await run(ch) finally: await disconnect(ch) diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index f69e3f9396..5f5abf8bb3 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -9,7 +9,7 @@ from datetime import date, timedelta from typing import Any -from server.cli._ch import connect_declared, disconnect, reads +from server.cli._ch import check_acceptance, connect_verified, disconnect, reads from server.core.datetimes import utc_now as _now # The first botocore whose put_object takes IfNoneMatch (the `aws` extra's floor). @@ -90,15 +90,20 @@ def new_client() -> Any: check_credentials(role_arn=args.role_arn, region=args.region) put = s3_put(new_client, args.bucket, expected_owner=args.expected_bucket_owner) - # No model calls and no writes to ClickHouse: a plain connection, outside - # the provider barrier that `run_connected` enters. It connects inside the - # deadline's scope, so the driver's read timeout is the deadline too. + # Analytics export consumes org data even without inference; its entire native + # body participates in permanent retirement without taking model admission. + from server.insights.app.provider_command import participant_command + async def _go() -> list[Exported]: with statement_timeout(STATEMENT_TIMEOUT_S): ch = get_client() - await connect_declared(ch, org_free=reads("reads; writes only the export bucket")) + await connect_verified(ch) try: - return await export_day(day, prefix=args.prefix, put=put, name_key=name_key, ch=ch) + async with participant_command(): + await check_acceptance(org_free=reads("reads; writes only the export bucket")) + return await export_day( + day, prefix=args.prefix, put=put, name_key=name_key, ch=ch + ) finally: await disconnect(ch) diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 5479c3d851..1074951fa1 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -117,10 +117,11 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) try: # A repository added from the Repositories page lives in the registry, not in # LOOM_CODE_REPOS_JSON: read it before the subcommand resolves its namespace. + from server.cli._ch import connect_verified from server.code.app.index.repo_registry import REGISTRY_UNREAD_MESSAGE, refresh_db_repos from server.code.domain.index.repos import RegistryUnread - await connect_declared(get_client(), org_free=reads("reads the repository registry")) + await connect_verified(get_client()) async with model_command(): # Never raises; while it has never succeeded, `configured_repos` refuses. await refresh_db_repos(force=True) diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 60f3fdc6fc..983ce671cd 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -716,15 +716,16 @@ async def _clickgap_write_category_links(w: _ClickgapWrite) -> None: async def _clickgap_enroll(stack: Any, org: str) -> Any: """Connect and enroll before selecting the importer's effective embedder.""" from server import boot - from server.cli._ch import connect_checked + from server.cli._ch import check_acceptance, connect_verified from server.core.background import register_buffer_drainer from server.core.ch import get_client from server.embedding import get_embedder from server.insights.app.provider_command import model_command - await connect_checked(get_client(), orgs=(org,)) + await connect_verified(get_client()) register_buffer_drainer("boot-writes", boot.drain_writes) await stack.enter_async_context(model_command()) + await check_acceptance(orgs=(org,)) return get_embedder() diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index fd0925accb..b24c9371b5 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -781,22 +781,29 @@ async def _http_writer(op: str, payload: dict[str, Any]) -> None: return _http_writer, client.aclose - from server.cli._ch import connect_checked, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client handlers = inproc_ops() ch = get_client() - await connect_checked(ch, orgs=(args.org,)) + await connect_verified(ch) from server import boot from server.core.background import register_buffer_drainer from server.insights.app.provider_command import model_command register_buffer_drainer("boot-writes", boot.drain_writes) command = model_command() + entered = False try: await command.__aenter__() + entered = True + await check_acceptance(orgs=(args.org,)) except BaseException: - await disconnect(ch) + try: + if entered: + await command.__aexit__(None, None, None) + finally: + await disconnect(ch) raise async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 53cb8e4004..9a6cdb2437 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -207,8 +207,9 @@ def _cmd_reextract(args: argparse.Namespace) -> int: the stored transcripts, the source untouched.""" import asyncio - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command from server.memory.app.reextract import load_sessions, render_report, replay if args.into == args.namespace: @@ -225,33 +226,37 @@ def _cmd_reextract(args: argparse.Namespace) -> int: async def _go() -> int: ch = get_client() - await connect_declared( - ch, org_free=reads("reads; the replay writes through the server at --base-url") - ) + await connect_verified(ch) try: - sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + async with participant_command(): + await check_acceptance( + org_free=reads("reads; replay writes through the HTTP server") + ) + sessions = await load_sessions(ch, org=args.org, namespace=args.namespace) + if args.limit_sessions: + sessions = sessions[: args.limit_sessions] + if not sessions: + print( + f" no set_from_messages evidence in {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + rep = await replay( + sessions, + base_url=args.base_url, + token=token, + org=args.org, + into=args.into, + dates=args.dates, + concurrency=args.concurrency, + dry_run=args.dry_run, + ) + print( + render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run) + ) + return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: await disconnect(ch) - if args.limit_sessions: - sessions = sessions[: args.limit_sessions] - if not sessions: - print( - f" no set_from_messages evidence in {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - rep = await replay( - sessions, - base_url=args.base_url, - token=token, - org=args.org, - into=args.into, - dates=args.dates, - concurrency=args.concurrency, - dry_run=args.dry_run, - ) - print(render_report(rep, source=args.namespace, into=args.into, dry_run=args.dry_run)) - return 1 if rep.failed and rep.failed == rep.sessions else 0 return asyncio.run(_go()) diff --git a/server/cli/participant.py b/server/cli/participant.py index e42832b6af..9154bf4be4 100644 --- a/server/cli/participant.py +++ b/server/cli/participant.py @@ -1,10 +1,9 @@ """`loom participant retire`: the manual recovery for a Loom process that died without shutting down. -Every Loom process registers a provider participant row, and a provider -reconfiguration waits until each live participant acknowledges the drain. A -participant that stops beating expires by itself; this retires one at once, -once an operator has confirmed its process is gone. +A participant retires only after permanent guarded shutdown or verifiable exact +physical death on a trusted host, with its cross-replica queries settled. +Heartbeat age and an operator affirmation are not evidence of death. """ from __future__ import annotations @@ -17,8 +16,7 @@ def _cmd_retire(args: argparse.Namespace) -> int: - """Retire a provider participant row whose process died without shutting - down, so a provider reconfiguration need not wait for it to expire.""" + """Verify exact process death and retire the participant after its queries settle.""" from uuid import UUID from server.cli._ch import run_connected @@ -28,13 +26,6 @@ def _cmd_retire(args: argparse.Namespace) -> int: except ValueError: print(f" ✗ {args.participant!r} is not a participant id", file=sys.stderr) return 2 - if not args.yes: - print( - " ✗ retiring a participant that is still running lets a reconfiguration proceed " - "under its writes; confirm its process is stopped, then pass --yes", - file=sys.stderr, - ) - return 2 async def _go(ch: Any) -> str: from server.insights.app.provider_barrier import retire_participant @@ -47,8 +38,9 @@ async def _go(ch: Any) -> str: return 1 if outcome == "live": print( - f" ✗ participant {participant} is still running: its heartbeat is fresh or " - "a query of its own still runs", + f" ✗ participant {participant} has no verifiable exact physical-death proof " + "or its cross-replica queries have not settled; run on its trusted host after " + "the exact process stops. Expiry and Kubernetes object deletion are not proof", file=sys.stderr, ) return 1 @@ -61,9 +53,8 @@ def register_participant_cli(sub: Any) -> None: ssub = p.add_subparsers(dest="participant_cmd", required=True) r = ssub.add_parser( "retire", - help="Retire the provider participant of a process that died without shutting " - "down (its id is `participant_id` in that process's /v1/health).", + help="Verify exact physical death on this trusted host and retire its participant " + "after all queries settle (the id is participant_id in /v1/health).", ) r.add_argument("--participant", required=True, help="The participant id to retire") - r.add_argument("--yes", action="store_true", help="Its process is confirmed stopped") r.set_defaults(func=_cmd_retire) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 8499eed5cd..6a727c9e7b 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -70,14 +70,17 @@ def _connected( ) -> Any: """Run `work(ch)` on one plain connection: the command calls no model. `orgs` or `org_free` as `server.cli._ch.connect_declared` takes them.""" - from server.cli._ch import connect_declared, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client + from server.insights.app.provider_command import participant_command async def _go() -> Any: ch = get_client() - await connect_declared(ch, orgs=orgs, org_free=org_free) + await connect_verified(ch) try: - return await work(ch) + async with participant_command(): + await check_acceptance(orgs=orgs, org_free=org_free) + return await work(ch) finally: await disconnect(ch) diff --git a/server/core/cell.py b/server/core/cell.py index 2aaea0f529..a37ce9e029 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -272,6 +272,12 @@ def state(self) -> str: _awaiting_first_read = False +def discard_inherited_view() -> None: + """A child enrolls and reads primary state before using acceptance state.""" + global _view, _memo, _awaiting_first_read + _view, _memo, _awaiting_first_read = _View(), None, True + + def expect_first_read() -> None: global _awaiting_first_read _awaiting_first_read = True diff --git a/server/core/instances.py b/server/core/instances.py index 27a35d8ec6..ecbe666c07 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -142,9 +142,12 @@ def cell_row() -> dict[str, Any]: `marked` (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" + from server.core.provider_process import process_identity + reported = { **cell.as_value(cell.mine()), "serves": bool(settings.serves_traffic), + "participant_id": str(process_identity()), "marked": cell.marked(), **cell_router.router_key(), } diff --git a/server/core/provider_process.py b/server/core/provider_process.py index 6ef0d6af5a..a2c6ad6328 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -5,11 +5,9 @@ still has a query running. """ -import asyncio import os import re import socket -from pathlib import Path from uuid import UUID, uuid4 _identity: tuple[int, UUID] | None = None @@ -99,11 +97,14 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace), or it ran in this container - before the container restarted into a new pid namespace (which kills every - process of the old one). Two containers that share a host name never prove - each other stopped; a process that cannot be inspected is not proven stopped.""" + the one that started then (same pid namespace). Another namespace or API + object proves nothing. Sharing a host name or container identity alone does + not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") + if len(parts) == 9: + if parts[8] != LIFECYCLE: + return False + parts = parts[:8] if len(parts) not in (6, 7, 8): return False host, boot, container, pidns, pid, start = parts[:6] @@ -114,7 +115,7 @@ def proves_stopped(proof: str) -> bool: if host != socket.gethostname() or boot != my_boot: return False if pidns != my_pidns: - return bool(container) and container == _container() + return False # namespace identity alone says nothing about old processes # Only positive evidence: no such process, or another one under its pid. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: @@ -129,45 +130,36 @@ def proves_stopped(proof: str) -> bool: return False -_SA = "/var/run/secrets/kubernetes.io/serviceaccount" -_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") +LIFECYCLE = "cell-retirement-v1" -async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: - """Whether Kubernetes says the pod `proof` names is gone: no pod of that name - in its namespace, one with another UID, or this one in phase Failed or - Succeeded (an evicted pod keeps its object; a pod that restarts its - containers reaches those phases only once every one stopped for good). The - kubelet removes a pod's object once its containers stopped (or an operator - force-deleted it), so each is positive evidence a lost pod's process ended. - Asked only of this pod's own namespace, with its service-account token; - False wherever the API or the proof does not say.""" +def certified(proof: str) -> str: + """Preserve the exact captured identity and stamp the installed protocol.""" parts = proof.split("|") - found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None - namespace = os.environ.get("POD_NAMESPACE", "") - if not found or not namespace or parts[6] != namespace or not parts[7]: + if len(parts) == 9 and parts[8] == LIFECYCLE: + parts = parts[:8] + if len(parts) not in (6, 7, 8): + raise ValueError("Unrecognized participant identity proof") + parts += [""] * (8 - len(parts)) + if not all(parts[i] for i in (0, 1, 3, 4, 5)): + raise ValueError("Incomplete participant identity proof") + if not parts[4].isdecimal() or int(parts[4]) <= 0 or not parts[5].isdecimal(): + raise ValueError("Invalid participant process identity") + return "|".join((*parts, LIFECYCLE)) + + +def has_certificate(proof: str) -> bool: + if len(proof.split("|")) != 9: return False - host = os.environ.get("KUBERNETES_SERVICE_HOST", "") - port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") try: - token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() - except OSError: - return False - if not host or not token: + return certified(proof) == proof + except ValueError: return False - import httpx - url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" - try: - async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: - r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) - except (httpx.HTTPError, OSError): - return False - if r.status_code == 404: - return True - if r.status_code != 200: - return False - pod = r.json() - if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): - return True - return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") + +def participant_proof() -> str: + """Advertise capability only in a composition with installed lifecycle guards.""" + from server.core.retirement import get_gate + + proof = host_proof() + return certified(proof) if proof and get_gate().role else proof diff --git a/server/core/retirement.py b/server/core/retirement.py new file mode 100644 index 0000000000..025a437362 --- /dev/null +++ b/server/core/retirement.py @@ -0,0 +1,114 @@ +"""Permanent process admission and the evidence passed to the retirement writer. + +Model maintenance can reopen. This gate never does: the participant identity is +finished once shutdown begins, even when its shutdown budget expires. +""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from server.core import provider_process + + +class ProcessStopping(RuntimeError): + """This participant can never accept another unit of work.""" + + +class RetirementGate: + """One event loop; checking admission and incrementing never await.""" + + def __init__(self) -> None: + self.active = 0 + self.closed = False + self.role = "" + self.enrolled = False + self.producers_stopped = False + self.producers: set[asyncio.Task[Any]] = set() + + def install(self, role: str) -> None: + if self.closed or self.role not in ("", role): + raise ProcessStopping("A retired participant cannot enroll again") + self.role = role + + def close(self) -> None: + self.closed = True + + def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: + """A process-lifetime producer must actually end, not merely be canceled.""" + self.producers.add(task) + task.add_done_callback(self.producers.discard) + if self.closed: + task.cancel() + raise ProcessStopping("Cannot start a producer during shutdown") + return task + + def stopped(self) -> None: + """Composition calls this after stopping every role-specific producer.""" + self.close() + self.producers_stopped = True + + @property + def quiescent(self) -> bool: + return ( + self.closed + and bool(self.role) + and self.producers_stopped + and self.active == 0 + and all(task.done() for task in self.producers) + ) + + @asynccontextmanager + async def admit(self) -> AsyncIterator[None]: + if self.closed or (self.role and not self.enrolled): + raise ProcessStopping("This process is stopping or has not enrolled") + self.active += 1 + try: + yield + finally: + self.active -= 1 + + +_pid = os.getpid() +_gate = RetirementGate() + + +def get_gate() -> RetirementGate: + """A fork cannot inherit an accepting view or its parent's retirement state.""" + global _pid, _gate + if _pid != os.getpid(): + role = _gate.role + _pid, _gate = os.getpid(), RetirementGate() + _gate.role = role + return _gate + + +@dataclass(frozen=True) +class RetirementProof: + """Internal evidence; only guarded shutdown or verified death constructs it.""" + + participant: UUID + host_proof: str + after: int + version: int | None = None + + +def guarded_proof(participant: UUID, *, version: int) -> RetirementProof: + gate = get_gate() + proof = provider_process.host_proof() + if not gate.quiescent or not gate.enrolled or not proof: + raise ProcessStopping("Permanent guarded quiescence is not established") + return RetirementProof(participant, provider_process.certified(proof), version - 1, version) + + +def death_proof(participant: UUID, proof: str, *, after: int) -> RetirementProof | None: + """Called only after the complete cross-replica running-query census is zero.""" + if not provider_process.proves_stopped(proof): + return None + return RetirementProof(participant, provider_process.certified(proof), after) diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 503cd021e2..193fcc9335 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -11,6 +11,7 @@ import asyncio import contextlib import logging +import os import threading from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager @@ -18,10 +19,11 @@ from typing import Any from uuid import UUID +from server.core import retirement from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import query_tag from server.core.model_work_gate import get_model_work_gate -from server.core.provider_process import host_proof, pod_gone, process_identity, proves_stopped +from server.core.provider_process import participant_proof, process_identity, proves_stopped from server.insights.store.provider_participants import ( OWN_VERSION_SQL, SEQUENTIAL, @@ -30,6 +32,7 @@ participants, prune_retired, record_participant, + record_retirement, running_queries, ) @@ -62,6 +65,8 @@ def stamp( ) -> tuple[UUID, str, int]: with self.lock: if transition is not None and state is not None: + if self.state == "retired" and state != "retired": + raise retirement.ProcessStopping("A retired UUID cannot re-enroll") self.transition, self.state = transition, state self.version = max(time_ns(), self.version + 1, floor + 1) return self.transition, self.state, self.version @@ -109,7 +114,7 @@ def beat(self) -> None: transition, state, version = _OWN.stamp(floor=seen) if state == "retired": return - row = [me, transition, state, version, STALE_AFTER_S, host_proof()] + row = [me, transition, state, version, STALE_AFTER_S, participant_proof()] self.ch.insert(TABLE, [row], column_names=_COLUMNS, settings=_TAG) def _run(self) -> None: @@ -146,6 +151,7 @@ def signal_stop(self) -> None: _HEARTBEAT = _Heartbeat() _registered = False +_process_pid = os.getpid() async def register_process() -> None: @@ -155,11 +161,20 @@ async def register_process() -> None: startup loads the pending gate or the committed replacement before admission. A second call (boot enrolls before its migrations) keeps the state. """ - global _registered + global _registered, _process_pid, _OWN, _HEARTBEAT + if _process_pid != os.getpid(): + _process_pid = os.getpid() + _registered, _OWN, _HEARTBEAT = False, _Own(), _Heartbeat() + from server.core import cell + + cell.discard_inherited_view() + if retirement.get_gate().closed: + raise retirement.ProcessStopping("A retired participant cannot re-enroll") if not _registered: _OWN.stamp(UUID(int=0), "running") await asyncio.to_thread(_HEARTBEAT.start) _registered = True + retirement.get_gate().enrolled = True async def enroll_before_migrations(ch: Any) -> None: @@ -216,18 +231,24 @@ async def distributed_drain(transition: UUID) -> bool: async def retire_process() -> bool: - """Retire only when no admitted or registered deferred writer remains, and no - write of this process can still land on the server.""" - global _registered - if get_model_work_gate().active or not await drain_task_sets(timeout_s=0): + """Permanently close admission; certify only actually completed role work.""" + gate = retirement.get_gate() + gate.close() + _HEARTBEAT.signal_stop() + if not gate.quiescent or get_model_work_gate().active: + return False + if not await drain_task_sets(timeout_s=0): + return False + if not await flush_registered_buffers() or not await drain_task_sets(timeout_s=0): return False if not await _server_side_settled(): return False - _HEARTBEAT.signal_stop() + if _HEARTBEAT.running or await running_queries(str(process_identity())) != 0: + return False _, _, version = _OWN.stamp(UUID(int=0), "retired") + proof = retirement.guarded_proof(process_identity(), version=version) with query_tag(_TAG["log_comment"]): - await record_participant(process_identity(), UUID(int=0), "retired", version=version) - _registered = False + await record_retirement(proof) return True @@ -256,7 +277,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " "or an insert of this process still running; retire it with " - "`loom participant retire --participant %s --yes` once it stopped", + "`loom participant retire --participant %s` on its trusted host once it stopped", rmt_carry.unsettled(), process_identity(), ) @@ -268,7 +289,7 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: - """A boot that fails after enrolling retires its row instead of leaving one per attempt.""" + """A failed boot stops known producers but never certifies an incomplete shutdown.""" @asynccontextmanager async def wrapped(app: T) -> AsyncIterator[None]: @@ -279,6 +300,10 @@ async def wrapped(app: T) -> AsyncIterator[None]: yield except BaseException: if not started and _registered: + gate = retirement.get_gate() + gate.close() + for producer in tuple(gate.producers): + producer.cancel() try: await retire_process() except Exception as e: # noqa: BLE001 — the row expires by heartbeat instead @@ -299,11 +324,16 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: return "absent" if str(participant) == str(process_identity()): return "live" - if not row.get("expired"): + if not proves_stopped(str(row.get("host_proof") or "")): return "live" if await running_queries(participant, ch=ch) != 0: return "live" - await record_participant(participant, UUID(int=0), "retired", after=int(row["version"]), ch=ch) + proof = retirement.death_proof( + participant, str(row.get("host_proof") or ""), after=int(row["version"]) + ) + if proof is None: + return "live" + await record_retirement(proof, ch=ch) return "retired" @@ -328,9 +358,12 @@ async def retire_dead_here(*, now: float | None = None) -> int: continue if await running_queries(participant) != 0: continue - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) + if proof is None: + continue + await record_retirement(proof) log.info("provider participant %s retired: its process on this host is gone", participant) retired += 1 return retired @@ -348,10 +381,7 @@ def stale_participants() -> tuple[dict[str, str], ...] | None: async def sweep_stopped(*, now: float | None = None) -> None: - """Retire the expired participants with no query left whose Kubernetes pod is - gone (`pod_gone`), and remember the rest that block a reconfiguration (only - their process, Kubernetes or an operator can say they stopped). The workers - leader's job.""" + """Remember stale blockers; only exact verified death authorizes retirement.""" global _STALE, _last_sweep, _last_prune t = monotonic() if now is None else now if _STALE is not None and t - _last_sweep < SWEEP_S: @@ -369,11 +399,15 @@ async def sweep_stopped(*, now: float | None = None) -> None: entry = {"participant_id": participant, "heartbeat_at": str(row.get("heartbeat_at", ""))} if row.get("expired"): running = await running_queries(participant) != 0 - if not running and await pod_gone(str(row.get("host_proof") or "")): - await record_participant( - UUID(participant), UUID(int=0), "retired", after=int(row["version"]) + proof = ( + None + if running + else retirement.death_proof( + UUID(participant), str(row.get("host_proof") or ""), after=int(row["version"]) ) - log.info("provider participant %s retired: Kubernetes deleted its pod", participant) + ) + if proof is not None: + await record_retirement(proof) continue stale.append({**entry, "reason": "queries_running" if running else "heartbeat_expired"}) _STALE = tuple(stale) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 46ec2bb9a5..b89c418e17 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -6,7 +6,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager -from server.core import settings_overrides +from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers from server.core.ch import set_query_tag from server.core.model_work_gate import get_model_work_gate @@ -16,6 +16,8 @@ log = logging.getLogger(__name__) +COMMAND_DRAIN_S = 15.0 # bounded cleanup; timeout leaves the participant unretired + async def _adopt_overrides(*, failing: bool = False) -> bool: """Whether the stored overrides loaded. A failure is logged when a run of them @@ -41,22 +43,44 @@ async def _keep_overrides( @asynccontextmanager -async def model_command() -> AsyncIterator[None]: - """A running command blocks a rebuild; a newborn loads the gate first. The - deployment's live-setting overrides apply before the work starts, as they do - in every server process, so a native command and the HTTP op agree.""" +async def participant_command() -> AsyncIterator[None]: + """Register all native org/principal work, including commands without models.""" + retirement.get_gate().install("command") await register_process() - keeper: asyncio.Task[None] | None = None try: - await load_active_configuration() - loaded = await _adopt_overrides() - keeper = asyncio.create_task(_keep_overrides(loaded), name="loom-cli-settings-overrides") - async with get_model_work_gate().admit(): + async with retirement.get_gate().admit(): yield finally: - if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper - if await drain_task_sets(timeout_s=15) and await flush_registered_buffers(): + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + embedder_calibration.stop_refresh_loop() + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + if producers: + await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + gate.stopped() + if await drain_task_sets(timeout_s=COMMAND_DRAIN_S) and await flush_registered_buffers(): await retire_process() + + +@asynccontextmanager +async def model_command() -> AsyncIterator[None]: + """The permanent lifecycle also holds the reversible model-work admission.""" + async with participant_command(): + keeper: asyncio.Task[None] | None = None + try: + await load_active_configuration() + loaded = await _adopt_overrides() + keeper = asyncio.create_task( + _keep_overrides(loaded), name="loom-cli-settings-overrides" + ) + async with get_model_work_gate().admit(): + yield + finally: + if keeper is not None: + keeper.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keeper diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 256a40babb..da04df218d 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -6,12 +6,18 @@ from server.core.ch import sequential_reads from server.core.ch_errors import error_code -from server.core.provider_process import host_proof, participant_tag, process_identity +from server.core.provider_process import ( + has_certificate, + participant_proof, + participant_tag, + process_identity, +) from server.core.repo_base import _CHClient, _default_client +from server.core.retirement import RetirementProof, get_gate from server.settings import settings TABLE = "loom._provider_participants" -# A participant whose last heartbeat is older than this may count as stopped. +# Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 # `expired` is decided on the server's clock, like the leases. @@ -19,7 +25,7 @@ "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL WHERE state != 'retired'" + f"FROM {TABLE} FINAL" ) ONE_SQL = ( "SELECT state, version, " @@ -58,7 +64,12 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id FROM {TABLE} FINAL WHERE state = 'retired' " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL WHERE state = 'retired' " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$') " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" @@ -66,7 +77,12 @@ def running_sql(*, inserts: bool = False) -> str: async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: with sequential_reads(): - return await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + return [ + r + for r in rows + if r["state"] != "retired" or not has_certificate(str(r.get("host_proof") or "")) + ] async def record_participant( @@ -80,9 +96,13 @@ async def record_participant( ) -> None: """`heartbeat_at` takes the server's clock (its column default); this process's own row carries its `host_proof`.""" - if state not in ("running", "drained", "retired"): + if state not in ("running", "drained"): raise ValueError("Unknown participant state") own = str(identity) == str(process_identity()) + if not own: + raise ValueError("A process writes only its own running/drained registration") + if get_gate().closed: + raise ValueError("A permanently closed participant cannot re-enroll") await (ch or _default_client()).insert( TABLE, [ @@ -94,21 +114,58 @@ async def record_participant( # A retired row is stopped at once, so every build's `provider-lock clear` # frees its holder without waiting out the heartbeat window. "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": host_proof() if own else "", + "host_proof": participant_proof() if own else "", + } + ], + ) + + +async def record_retirement(proof: RetirementProof, *, ch: _CHClient | None = None) -> None: + """The sole retired-row writer, passed evidence from the guarded lifecycle.""" + if not isinstance(proof, RetirementProof) or not has_certificate(proof.host_proof): + raise ValueError("Retirement requires verified process evidence") + await (ch or _default_client()).insert( + TABLE, + [ + { + "participant_id": proof.participant, + "transition_id": UUID(int=0), + "state": "retired", + "version": max(time_ns(), proof.after + 1, proof.version or 0), + "stale_after_s": 0, + "host_proof": proof.host_proof, } ], ) async def prune_retired(*, ch: _CHClient | None = None) -> int: - """Delete every row of the participants retired `PRUNE_AFTER_H` ago (each - process start leaves one); a process that was only paused enrolls again - with its next beat. Returns how many.""" + """Age selects already certified permanent retirements, never unknown rows. + + Retired UUIDs cannot re-enroll. An earlier in-flight beat is below the final + version; removing every version cannot resurrect that beat. + """ client = ch or _default_client() - rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) - ids = [r["participant_id"] for r in rows] - if ids: - await client.execute(_PRUNE_SQL, {"ids": ids}) + with sequential_reads(): + rows = await client.query_all(_PRUNE_IDS_SQL, {"hours": PRUNE_AFTER_H}) + ids = [r["participant_id"] for r in rows if has_certificate(str(r.get("host_proof") or ""))] + if not ids: + return 0 + latest = await client.query_all( + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + expected = {str(r["participant_id"]): int(r["version"]) for r in rows} + ids = [ + r["participant_id"] + for r in latest + if r["state"] == "retired" + and int(r["version"]) == expected.get(str(r["participant_id"])) + and has_certificate(str(r.get("host_proof") or "")) + ] + if ids: + await client.execute(_PRUNE_SQL, {"ids": ids}) return len(ids) diff --git a/server/main.py b/server/main.py index 647e18b1e5..d7a3ffc2f2 100644 --- a/server/main.py +++ b/server/main.py @@ -29,6 +29,7 @@ ) from server.code.app.index.repo_modes import repo_umask from server.core import readiness as _readiness +from server.core import retirement from server.core.ch import get_client, probe_reader_user from server.core.errors import classified from server.core.logging_config import configure_logging @@ -46,6 +47,22 @@ wire() +def verify_retirement_topology() -> None: + """Capability requires both sides of the request-tail and child-task boundaries.""" + from server.core.request_tail import RequestTailMiddleware + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + classes: list[object] = [middleware.cls for middleware in app.user_middleware] + if not ( + bool(classes) + and all(cls in classes for cls in (RequestTailMiddleware, ModelMaintenanceMiddleware)) + and classes[0] is RetirementMiddleware + and classes.index(RequestTailMiddleware) < classes.index(ModelMaintenanceMiddleware) + ): + raise RuntimeError("Retirement middleware must wrap the request tail and application") + + async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None: """The bounded part of a shutdown: the instance row's stop write and the drains share one budget, and the final cleanup gets what they left unused, so the whole @@ -56,24 +73,35 @@ async def _shut_down(ch: Any, lease: Any, supervisor: Any, emitter: Any) -> None started = time.monotonic() await _instances.stop(ch) left = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) - try: - await asyncio.wait_for(_shutdown_drains(lease, supervisor, emitter), timeout=left) - except TimeoutError: + from server.core.background import create_deferred_task + + drains = create_deferred_task( + _shutdown_drains(lease, supervisor, emitter), name="shutdown-drains" + ) + _, pending = await asyncio.wait({drains}, timeout=left) + if pending: + drains.cancel() + logging.getLogger(__name__).warning( - "shutdown drains exceeded %.0f s; retiring anyway", boot.SHUTDOWN_DRAIN_BUDGET_S + "shutdown drains exceeded %.0f s; permanent retirement remains unproved", + boot.SHUTDOWN_DRAIN_BUDGET_S, ) + else: + await drains unused = max(0.0, boot.SHUTDOWN_DRAIN_BUDGET_S - (time.monotonic() - started)) await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S + unused) async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: """The lifespan's shutdown drains, in order; it bounds them all together.""" - from server.workers import leader as _leader - # An agent claim's last cutoff, which ends what an older replica's sign-in # with the code issued while the claim was landing: started first, so it # runs beside the drains below. - claims, started = asyncio.create_task(_agent_claim.drain()), time.monotonic() + from server.core.background import create_deferred_task + from server.workers import leader as _leader + + claims = create_deferred_task(_agent_claim.drain(), name="shutdown-agent-claims") + started = time.monotonic() try: await _leader.shutdown(lease, supervisor) # The service-side writes still in flight land while what they use is @@ -101,10 +129,29 @@ async def _shutdown_drains(lease: Any, supervisor: Any, emitter: Any) -> None: finally: claims.cancel() # a no-op once it is done await emitter.stop() + producers = tuple(retirement.get_gate().producers) + if producers: + await asyncio.gather(*producers, return_exceptions=True) _otel.shutdown() from server.clients import _http as _http_clients await _http_clients.aclose() + retirement.get_gate().stopped() + + +async def _stop_background(probe_listener: Any) -> None: + """Stop admission and signal every producer; retirement still waits for completion.""" + from server.code.app.index import mirrors + + retirement.get_gate().close() + _readiness.begin_drain() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() + if probe_listener is not None: + await probe_listener.stop() + mirrors.stop_warmup() + embedder_calibration.stop_refresh_loop() + boot.cancel_boot_tasks() @asynccontextmanager @@ -141,6 +188,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: weak_keys=weak_keys(), ) + verify_retirement_topology() + retirement.get_gate().install("server") + track = retirement.get_gate().track from server.core.object_store import require_client require_client() @@ -174,7 +224,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_storage_authority(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - cell_loops = await boot.start_cell_gate() + from server.insights.app.provider_barrier import register_process + + await register_process() + for task in await boot.start_cell_gate(): + track(task) await boot.warn_newer_schema() await boot.warn_collection_named_namespaces() # Nothing here writes a column a migration just added while another process @@ -187,21 +241,21 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from server.core import instances as _instances from server.workers import leader as _leader - instances_loop = await _instances.start( - ch, - _leader.leads_deployment, - label=_instance_mod.instance_label() or "", - workers_lease=_leader.advertised_lease, - on_beat=_leader.read_holder, + track( + await _instances.start( + ch, + _leader.leads_deployment, + label=_instance_mod.instance_label() or "", + workers_lease=_leader.advertised_lease, + on_beat=_leader.read_holder, + ) ) from server.core.background import register_buffer_drainer - from server.insights.app.provider_barrier import register_process register_buffer_drainer("boot-writes", boot.drain_writes) from server.insights.app.provider_runtime import load_active_configuration, refresh_loop - await register_process() await load_active_configuration() emitter = get_emitter() @@ -234,24 +288,29 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: boot.publish_deployment_config(ch) # This process verifies tokens: it loads the revoked set and the signing # keyring now and keeps both fresh. - auth_loops = await boot.start_token_state() + for task in await boot.start_token_state(): + track(task) probe_listener = boot.start_probe_listener() - overrides_loop = boot.start_runtime_overrides() + track(boot.start_runtime_overrides()) await probe_reader_user(ch) from server.insights.app import analysis as _analysis # Provisioned and probed in the background: the lease can wait for minutes. - analysis_loop = _analysis.start_probe_loop() + track(_analysis.start_probe_loop()) from server.assistant.app import service as _assistant assistant_loop = _assistant.start_setup_loop() - tracker_sources_loop = boot.start_tracker_sources_refresher(ch) + if assistant_loop is not None: + track(assistant_loop) + track(boot.start_tracker_sources_refresher(ch)) from server.code.app.index import repo_registry as _repo_registry - code_repos_loop = await _repo_registry.start_refresher(ch) + track(await _repo_registry.start_refresher(ch)) from server.code.app.index import mirrors as _mirrors - _mirrors.start_warmup() # this host's git mirrors, in the background + mirror_loop = _mirrors.start_warmup() + if mirror_loop is not None: + retirement.get_gate().track(mirror_loop) boot.publish_frontend_assets(ch) boot.mark_process_start(ch) await boot.probe_sandbox() @@ -269,37 +328,22 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: "WorkerSupervisor: not started in pid %s — %s", os.getpid(), lease.reason ) lease_loop = boot.start_lease_loop(lease, supervisor, ch) + if lease_loop is not None: + track(lease_loop) if not get_model_work_gate().closed: await boot.provider_report() await boot.probe_vector_capabilities(ch) await boot.calibrate_similarity_gates() - provider_refresh = asyncio.create_task(refresh_loop(), name="provider-configuration-refresh") + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")) # SIGTERM: not ready first, the port closes LOOM_SHUTDOWN_READY_DELAY_S later. boot.install_drain_on_sigterm() try: yield finally: - _readiness.begin_drain() - provider_refresh.cancel() - await asyncio.gather(provider_refresh, return_exceptions=True) - if lease_loop is not None: - lease_loop.cancel() - for loop in (*cell_loops, *auth_loops, overrides_loop): - loop.cancel() - if probe_listener is not None: - await probe_listener.stop() - tracker_sources_loop.cancel() - code_repos_loop.cancel() - analysis_loop.cancel() - if assistant_loop is not None: - assistant_loop.cancel() - _mirrors.stop_warmup() - embedder_calibration.stop_refresh_loop() - instances_loop.cancel() - boot.cancel_boot_tasks() + await _stop_background(probe_listener) await _shut_down(ch, lease, supervisor, emitter) await boot.close_storage_authority() @@ -495,6 +539,10 @@ def attach_browser_middleware(target: FastAPI) -> None: http_transport.attach(app) mcp_transport.attach(app) +from server.transport.retirement import RetirementMiddleware # noqa: E402 + +app.add_middleware(RetirementMiddleware) + from server.auth import login as _login # noqa: E402 from server.auth import oauth as _oauth # noqa: E402 from server.auth import scim_routes as _scim_routes # noqa: E402 diff --git a/server/memory/app/embedder_calibration.py b/server/memory/app/embedder_calibration.py index cc9569ba6b..8ddf1f4eb5 100644 --- a/server/memory/app/embedder_calibration.py +++ b/server/memory/app/embedder_calibration.py @@ -592,7 +592,11 @@ async def _loop() -> None: if not get_model_work_gate().closed: await refresh_shared() - _refresh_task = asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + from server.core.retirement import get_gate + + _refresh_task = get_gate().track( + asyncio.get_running_loop().create_task(_loop(), name="loom-calibration-refresh") + ) return _refresh_task diff --git a/server/observability/events.py b/server/observability/events.py index da4de2c333..a89e648a0c 100644 --- a/server/observability/events.py +++ b/server/observability/events.py @@ -194,8 +194,9 @@ async def start(self) -> None: # Process-lifetime task: it must not inherit the tag, request id or # auth of whichever request first started it. from server.core.background import detached_task + from server.core.retirement import get_gate - self._task = detached_task(self._drain_loop(), name="loom-events-drain") + self._task = get_gate().track(detached_task(self._drain_loop(), name="loom-events-drain")) async def stop(self) -> None: self._running = False diff --git a/server/observability/problems.py b/server/observability/problems.py index 07571de3c6..6e43eccfc8 100644 --- a/server/observability/problems.py +++ b/server/observability/problems.py @@ -936,8 +936,9 @@ def _participant_problems(state: HealthState) -> list[Problem]: f"A provider reconfiguration would wait on these participants: {ids}. " "`heartbeat_expired` ones stopped heartbeating (they may be paused), " "and `queries_running` ones did so with a query still running.", - "Once a participant's process is confirmed stopped, retire it with " - "`loom participant retire --participant --yes`.", + "On its trusted host, verify exact physical death and settled queries with " + "`loom participant retire --participant `. Expiry or a deleted pod " + "object does not prove death.", source="runtime", ) ] diff --git a/server/transport/model_maintenance.py b/server/transport/model_maintenance.py index 7f59ba2e2c..f27092672d 100644 --- a/server/transport/model_maintenance.py +++ b/server/transport/model_maintenance.py @@ -6,7 +6,7 @@ from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send -from server.core import cell, deployment_config +from server.core import cell, deployment_config, retirement from server.core.errors import classified from server.core.model_work_gate import ModelChangeInProgress, ModelWorkGate, get_model_work_gate from server.settings import settings @@ -111,6 +111,20 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] not in ("http", "websocket"): await self.app(scope, receive, send) return + try: + async with retirement.get_gate().admit(): + await self._dispatch(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) + + async def _dispatch(self, scope: Scope, receive: Receive, send: Send) -> None: if (reason := cell.gate_reason()) and not _health_probe(scope): await _refuse_cell(scope, receive, send, reason) return diff --git a/server/transport/retirement.py b/server/transport/retirement.py new file mode 100644 index 0000000000..396a304554 --- /dev/null +++ b/server/transport/retirement.py @@ -0,0 +1,28 @@ +"""Outer admission includes request-tail settlement after inner dispatch ends.""" + +from starlette.types import ASGIApp, Receive, Scope, Send + +from server.core import retirement +from server.transport.model_maintenance import _refuse + + +class RetirementMiddleware: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + try: + async with retirement.get_gate().admit(): + await self.app(scope, receive, send) + except retirement.ProcessStopping: + await _refuse( + scope, + receive, + send, + code="PROCESS_STOPPING", + message="This server is stopping. Retry another server.", + retry_after="1", + ) diff --git a/server/worker_main.py b/server/worker_main.py index 6854be4082..c265c7c124 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -20,6 +20,7 @@ from typing import Any from server import boot +from server.core import retirement from server.settings import settings from server.workers import child, leader from server.workers.supervisor import get_supervisor @@ -46,20 +47,24 @@ async def _trackers() -> None: # `with`, not set_query_tag: the supervisor and the shutdown run in this context after. with query_tag("boot.workers_child"): register_buffer_drainer("boot-writes", boot.drain_writes) + retirement.get_gate().install("worker") await register_process() await load_active_configuration() await get_emitter().start() await registry_sources.refresh_db_sources(force=True, ch=ch) if not get_model_work_gate().closed: await boot.calibrate_similarity_gates() + track = retirement.get_gate().track tasks = [ - asyncio.create_task(refresh_loop(), name="provider-configuration-refresh"), - asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh"), - await repo_registry.start_refresher(ch), + track(asyncio.create_task(refresh_loop(), name="provider-configuration-refresh")), + track(asyncio.create_task(_trackers(), name="loom-tracker-sources-refresh")), + track(await repo_registry.start_refresher(ch)), # The workers read LIVE settings too: their passes see the deployment's overrides. - boot.start_runtime_overrides(), + track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) + for task in tasks: + retirement.get_gate().track(task) return tasks @@ -113,13 +118,18 @@ async def _serve(fd: int, alive_fd: int) -> None: supervisor.start_all() await stop.wait() log.info("workers child %s stopping", os.getpid()) + retirement.get_gate().close() embedder_calibration.stop_refresh_loop() for task in services: task.cancel() await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() + retirement.get_gate().stopped() finally: + retirement.get_gate().close() + for producer in tuple(retirement.get_gate().producers): + producer.cancel() beat.cancel() await boot.leave_deployment(ch, budget_s=SHUTDOWN_RETIRE_S) await boot.close_storage_authority() diff --git a/server/workers/supervisor.py b/server/workers/supervisor.py index f951132452..b2b08946c5 100644 --- a/server/workers/supervisor.py +++ b/server/workers/supervisor.py @@ -30,6 +30,7 @@ from typing import Any from server.core.model_work_gate import get_model_work_gate +from server.core.retirement import ProcessStopping, get_gate from server.observability.traffic_class import bind_background from server.settings import settings from server.workers import leader, pass_context, status_store @@ -139,6 +140,7 @@ def _spawn(self, spec: WorkerSpec) -> None: _publish_streak(spec.name, 0) _publish_partial(spec.name, 0) task = asyncio.create_task(self._supervise_as_background(spec), name=_task_name(spec)) + get_gate().track(task) self._tasks.append(task) def _restart(self, spec: WorkerSpec, _ended: asyncio.Task[None]) -> None: @@ -257,7 +259,7 @@ async def _supervise(self, spec: WorkerSpec) -> None: try: # A pass past its deadline is cancelled and counted as a # failure (TimeoutError). - async with get_model_work_gate().admit(wait=True): + async with get_gate().admit(), get_model_work_gate().admit(wait=True): # Maintenance waiting consumes neither the pass deadline # nor its duration/accounting. An admitted pass finishes # normally; changing models never cancels it mid-write. @@ -278,6 +280,8 @@ async def _supervise(self, spec: WorkerSpec) -> None: spec, ctx, partial=bool(ctx.count), streak=consecutive_failures ) ) + except ProcessStopping: + return except pass_context.PassStopped: # The pass saw a stop or a stale lease between batches. Recorded # like a cancel; the loop top waits until the lease is fresh. diff --git a/tests/fixtures/provider_process_before_retirement.py b/tests/fixtures/provider_process_before_retirement.py new file mode 100644 index 0000000000..6ef0d6af5a --- /dev/null +++ b/tests/fixtures/provider_process_before_retirement.py @@ -0,0 +1,173 @@ +"""This process's provider-participant id and the tag its ClickHouse queries carry. + +Every ClickHouse client the process opens names the id in its HTTP User-Agent, +so `system.processes` says whether a participant that stopped heartbeating +still has a query running. +""" + +import asyncio +import os +import re +import socket +from pathlib import Path +from uuid import UUID, uuid4 + +_identity: tuple[int, UUID] | None = None + + +def process_identity() -> UUID: + """One id per process; a forked child draws its own.""" + global _identity + if _identity is None or _identity[0] != os.getpid(): + _identity = (os.getpid(), uuid4()) + return _identity[1] + + +def participant_tag(identity: UUID | str) -> str: + """What `system.processes.http_user_agent` contains for that participant's queries.""" + return f"loom-participant/{UUID(str(identity)).hex}" + + +def client_name() -> str: + """The `client_name` every ClickHouse client of this process is opened with.""" + return participant_tag(process_identity()) + + +# host name | kernel boot id | container | pid namespace | pid | process start +# (clock ticks after boot) | Kubernetes namespace and pod name (POD_NAMESPACE, +# POD_NAME: the host name is the pod name cut to 63 characters; '' elsewhere) +_proof: tuple[int, str] | None = None +# Docker mounts /etc/hostname from its container's directory; a Kubernetes container +# mounts its termination log from the pod's, under the container's name. +_CONTAINER = ( + re.compile(r"/containers/([0-9a-f]{64})/hostname "), + re.compile(r"/pods/([0-9a-f-]{36})/containers/([^/ ]+)/"), +) + + +def _start_ticks(pid: int | str) -> str: + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + return f.read().rsplit(")", 1)[1].split()[19] + + +def _boot_and_pidns() -> tuple[str, str]: + with open("/proc/sys/kernel/random/boot_id", encoding="ascii") as f: + boot = f.read().strip() + return boot, str(os.stat("/proc/self/ns/pid").st_ino) + + +def _container() -> str: + """This container's id, stable across its restarts; '' outside one, or where + the mounts do not say.""" + try: + with open("/proc/self/mountinfo", encoding="utf-8") as f: + mounts = f.read() + except OSError: + return "" + for pattern in _CONTAINER: + found = pattern.search(mounts) + if found: + return "/".join(found.groups()) + return "" + + +def host_proof() -> str: + """What lets a later process on this host prove this one stopped (`proves_stopped`); + '' where /proc does not say.""" + global _proof + if _proof is None or _proof[0] != os.getpid(): + try: + boot, pidns = _boot_and_pidns() + value = "|".join( + ( + socket.gethostname(), + boot, + _container(), + pidns, + str(os.getpid()), + _start_ticks("self"), + os.environ.get("POD_NAMESPACE", ""), + os.environ.get("POD_NAME", ""), + ) + ) + except (OSError, IndexError): + value = "" + _proof = (os.getpid(), value) + return _proof[1] + + +def proves_stopped(proof: str) -> bool: + """Whether the process `proof` names has stopped, judged from this one: same host + name and kernel boot, and either its pid is free or runs another process than + the one that started then (same pid namespace), or it ran in this container + before the container restarted into a new pid namespace (which kills every + process of the old one). Two containers that share a host name never prove + each other stopped; a process that cannot be inspected is not proven stopped.""" + parts = proof.split("|") + if len(parts) not in (6, 7, 8): + return False + host, boot, container, pidns, pid, start = parts[:6] + try: + my_boot, my_pidns = _boot_and_pidns() + except OSError: + return False + if host != socket.gethostname() or boot != my_boot: + return False + if pidns != my_pidns: + return bool(container) and container == _container() + # Only positive evidence: no such process, or another one under its pid. + # A pid this process may not inspect (hidepid, a permission error) proves nothing. + try: + os.kill(int(pid), 0) + except ProcessLookupError: + return True + except (OSError, ValueError): + return False + try: + return _start_ticks(pid) != start + except (OSError, IndexError): + return False + + +_SA = "/var/run/secrets/kubernetes.io/serviceaccount" +_POD_CONTAINER = re.compile(r"^([0-9a-f-]{36})/") + + +async def pod_gone(proof: str, *, timeout_s: float = 5.0) -> bool: + """Whether Kubernetes says the pod `proof` names is gone: no pod of that name + in its namespace, one with another UID, or this one in phase Failed or + Succeeded (an evicted pod keeps its object; a pod that restarts its + containers reaches those phases only once every one stopped for good). The + kubelet removes a pod's object once its containers stopped (or an operator + force-deleted it), so each is positive evidence a lost pod's process ended. + Asked only of this pod's own namespace, with its service-account token; + False wherever the API or the proof does not say.""" + parts = proof.split("|") + found = _POD_CONTAINER.match(parts[2]) if len(parts) == 8 else None + namespace = os.environ.get("POD_NAMESPACE", "") + if not found or not namespace or parts[6] != namespace or not parts[7]: + return False + host = os.environ.get("KUBERNETES_SERVICE_HOST", "") + port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") + try: + token = (await asyncio.to_thread(Path(f"{_SA}/token").read_text, "ascii")).strip() + except OSError: + return False + if not host or not token: + return False + import httpx + + url = f"https://{host}:{port}/api/v1/namespaces/{namespace}/pods/{parts[7]}" + try: + async with httpx.AsyncClient(verify=f"{_SA}/ca.crt", timeout=timeout_s) as client: + r = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + except (httpx.HTTPError, OSError): + return False + if r.status_code == 404: + return True + if r.status_code != 200: + return False + pod = r.json() + if str(pod.get("metadata", {}).get("uid", "")) != found.group(1): + return True + return str(pod.get("status", {}).get("phase", "")) in ("Failed", "Succeeded") diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py new file mode 100644 index 0000000000..0652b4403f --- /dev/null +++ b/tests/integration/test_cell_retirement.py @@ -0,0 +1,103 @@ +"""Certified retirement and pruning against an owned scratch ClickHouse table.""" + +from __future__ import annotations + +import asyncio +import os +import sys +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process, retirement +from server.core.ch import CHClient, query_tag +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( + monkeypatch, +): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + database = "retirement_" + uuid4().hex + table = database + ".participants" + client = CHClient() + await client.connect() + with query_tag("test.cell_retirement"): + await client.execute("CREATE DATABASE {database:Identifier}", {"database": database}) + try: + await client.execute( + "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", + {"database": database}, + ) + old_table = store.TABLE + for name in ( + "TABLE", + "PARTICIPANTS_SQL", + "OWN_VERSION_SQL", + "_PRUNE_IDS_SQL", + "_PRUNE_SQL", + ): + monkeypatch.setattr(store, name, getattr(store, name).replace(old_table, table)) + monkeypatch.setattr( + store, "_PRUNE_SQL", store._PRUNE_SQL + " SETTINGS mutations_sync=2" + ) + stopped, legacy = uuid4(), uuid4() + process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") + await asyncio.wait_for(process.wait(), timeout=10) + fields = provider_process.host_proof().split("|") + fields[4:6] = [str(process.pid), "1"] + identity = "|".join(fields) + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 1, + "host_proof": identity, + }, + {"participant_id": legacy, "state": "retired", "version": 1, "host_proof": ""}, + ], + ) + assert await store.running_queries(stopped, ch=client) == 0 + proof = retirement.death_proof(stopped, identity, after=1) + assert proof is not None + await store.record_retirement(proof, ch=client) + # A previously in-flight heartbeat cannot outrank the certified final row. + await client.insert( + table, + [ + { + "participant_id": stopped, + "state": "running", + "version": 2, + "host_proof": identity, + } + ], + ) + rows = await client.query_all( + f"SELECT participant_id, transition_id, state, version, stale_after_s, host_proof FROM {table} FINAL" + ) + final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) + assert str(final["transition_id"]) == str(UUID(int=0)) + assert final["state"] == "retired" and final["version"] > 2 + assert final["stale_after_s"] == 0 + assert final["host_proof"] == provider_process.certified(identity) + assert [str(row["participant_id"]) for row in await store.participants(ch=client)] == [ + str(legacy) + ] + # Both are old, but only the certified identity is allowed to disappear. + await client.execute( + f"ALTER TABLE {table} UPDATE heartbeat_at=now64(3)-toIntervalHour(25) " + "WHERE 1 SETTINGS mutations_sync=2" + ) + assert await store.prune_retired(ch=client) == 1 + rows = await client.query_all(f"SELECT participant_id, state FROM {table} FINAL") + assert [(str(row["participant_id"]), row["state"]) for row in rows] == [ + (str(legacy), "retired") + ] + finally: + await client.execute("DROP DATABASE {database:Identifier} SYNC", {"database": database}) + await client.disconnect() diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 9a6c9db1e2..16c9704fa8 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -19,11 +19,14 @@ def _live() -> None: @pytest.mark.asyncio -async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): +async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(monkeypatch): _live() ch = _default_client() await ch.connect() identity, transition = uuid4(), uuid4() + monkeypatch.setattr( + "server.insights.store.provider_participants.process_identity", lambda: identity + ) try: await record_participant(identity, transition, "running", ch=ch) await record_participant(identity, transition, "drained", ch=ch) @@ -39,10 +42,11 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(): rows = [r for r in await participants(ch=ch) if str(r["participant_id"]) == str(identity)] assert rows[0]["expired"] assert await running_queries(identity, ch=ch) == 0 - await record_participant( - identity, transition, "retired", after=int(rows[0]["version"]), ch=ch - ) - assert all(str(row["participant_id"]) != str(identity) for row in await participants(ch=ch)) + with pytest.raises(ValueError, match="state"): + await record_participant( + identity, transition, "retired", after=int(rows[0]["version"]), ch=ch + ) + assert any(str(row["participant_id"]) == str(identity) for row in await participants(ch=ch)) finally: try: await ch.execute( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ad5291a171..7a1f0ded28 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -80,6 +80,7 @@ async def checked(_ch): return None monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(provider_command, "participant_command", command) monkeypatch.setattr(pg_profile, "check_configured", checked) @@ -1152,3 +1153,11 @@ async def _none(org: str, namespaces: list[str], **_: object) -> dict[str, str]: return {} monkeypatch.setattr("server.knowledge.app.mirrors.mirrors_of", _none) + + +@pytest.fixture(autouse=True) +def isolated_retirement_gate(monkeypatch): + """Each test models a fresh process; permanent shutdown cannot leak between them.""" + from server.core import retirement + + monkeypatch.setattr(retirement, "_gate", retirement.RetirementGate()) diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 2567a90eb5..37e0e5e39e 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -185,12 +185,12 @@ ], "loom participant": [ "usage: loom participant [-h] {retire} ...", - "positional arguments: {retire} retire Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "positional arguments: {retire} retire Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "options: -h, --help show this help message and exit" ], "loom participant retire": [ - "usage: loom participant retire [-h] --participant PARTICIPANT [--yes]", - "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire --yes Its process is confirmed stopped" + "usage: loom participant retire [-h] --participant PARTICIPANT", + "options: -h, --help show this help message and exit --participant PARTICIPANT The participant id to retire" ], "loom deployment": [ "usage: loom deployment [-h] {publish} ...", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 2ad62a0df9..8216d4de36 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -5682,8 +5682,7 @@ "description": null, "dests": [ "help", - "participant", - "yes" + "participant" ] } ], @@ -5719,21 +5718,6 @@ "required": true, "help": "The participant id to retire", "metavar": null - }, - { - "option_strings": [ - "--yes" - ], - "dest": "yes", - "action": "_StoreTrueAction", - "default": false, - "const": true, - "choices": null, - "nargs": 0, - "type": null, - "required": false, - "help": "Its process is confirmed stopped", - "metavar": null } ] } @@ -5744,7 +5728,7 @@ "choices_help": [ { "dest": "retire", - "help": "Retire the provider participant of a process that died without shutting down (its id is `participant_id` in that process's /v1/health).", + "help": "Verify exact physical death on this trusted host and retire its participant after all queries settle (the id is participant_id in /v1/health).", "metavar": "retire" } ] diff --git a/tests/unit/test_analytics_export.py b/tests/unit/test_analytics_export.py index 7967d8edec..79412dc567 100644 --- a/tests/unit/test_analytics_export.py +++ b/tests/unit/test_analytics_export.py @@ -252,7 +252,10 @@ async def disconnect(self) -> None: def test_an_export_of_yesterday_connects_inside_its_deadline_and_hashes_under_the_key( - now: datetime, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + stub_model_command, + now: datetime, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: from server.core import ch as ch_module from server.insights.app import analytics_export as app diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 60abc95312..6d14350e03 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -17,7 +17,13 @@ from server.settings import settings ROOT = Path(__file__).resolve().parents[2] -_CONNECTS = {"run_connected", "connect_checked", "connect_declared", "_connected"} +_CONNECTS = { + "run_connected", + "connect_checked", + "connect_declared", + "check_acceptance", + "_connected", +} # `.connect()` receivers that are not the deployment's ClickHouse writer: the setup # command's own admin client and SQLite files an importer reads. _OTHER_CLIENTS = {"preflight", "sqlite3"} @@ -969,6 +975,7 @@ def test_a_cell_whose_rows_cannot_be_read_refuses_every_declared_org( asyncio.run(_ch.check_orgs([])) +@pytest.mark.usefixtures("stub_model_command") def test_a_command_for_an_org_the_cell_does_not_hold_never_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py new file mode 100644 index 0000000000..a5a3c8dcce --- /dev/null +++ b/tests/unit/test_cell_retirement.py @@ -0,0 +1,752 @@ +"""Permanent retirement requires completion, never expiry or an API object's absence.""" + +from unittest.mock import AsyncMock +from uuid import UUID, uuid4 + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store + + +@pytest.mark.asyncio +async def test_expired_quiet_participant_is_not_manual_death(monkeypatch): + participant = uuid4() + row = { + "participant_id": participant, + "transition_id": UUID(int=0), + "version": 7, + "state": "running", + "expired": True, + "host_proof": provider_process.host_proof(), + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_participant", AsyncMock()) + assert await barrier.retire_participant(participant) == "live" + barrier.record_participant.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_store_cannot_manufacture_retirement(): + ch = AsyncMock() + with pytest.raises(ValueError, match="[Rr]etire|[Ss]tate"): + await store.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) + ch.insert.assert_not_awaited() + + +def test_changed_namespace_does_not_prove_old_container_ended(monkeypatch): + fields = provider_process.host_proof().split("|") + assert len(fields) == 8 + fields[2] = "c" * 64 + fields[3] = "another-namespace" + monkeypatch.setattr(provider_process, "_container", lambda: fields[2]) + assert not provider_process.proves_stopped("|".join(fields)) + + +@pytest.mark.asyncio +async def test_close_is_permanent_through_an_awaited_retirement_write(monkeypatch): + import asyncio + + from server.core import retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + entered, finish = asyncio.Event(), asyncio.Event() + + async def write(proof): + assert gate.quiescent + entered.set() + await finish.wait() + + monkeypatch.setattr(barrier, "record_retirement", write) + monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + task = asyncio.create_task(barrier.retire_process()) + await entered.wait() + with pytest.raises(retirement.ProcessStopping): + async with gate.admit(): + pytest.fail("Permanent closure must not reopen while insert is awaited") + with pytest.raises(retirement.ProcessStopping): + await barrier.register_process() + finish.set() + assert await task + + +def _request_app(handler): + from fastapi import FastAPI + + from server.main import attach_browser_middleware + from server.transport import http + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + app = FastAPI() + app.add_api_route("/oauth/token", handler, methods=["POST"]) + app.add_middleware(ModelMaintenanceMiddleware) + attach_browser_middleware(app) + http.attach(app) + app.add_middleware(RetirementMiddleware) + return app + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_tail", [False, True]) +async def test_composed_request_tail_holds_counter_after_dispatch_raises(monkeypatch, cancel_tail): + import asyncio + + import httpx + + from server.core import request_tail, retirement + + gate = retirement.get_gate() + entered, finish, canceled = asyncio.Event(), asyncio.Event(), asyncio.Event() + observed = [] + + async def write(): + observed.append(gate.active) + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() # cancellation requested; this writer still exists + + async def handler(): + assert gate.active == 2 + assert request_tail.defer(write) + raise ValueError("before response start") + + app = _request_app(handler) + transport = httpx.ASGITransport(app=app, raise_app_exceptions=False) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient(transport=transport, base_url="http://localhost") as client: + task = asyncio.create_task(client.post("/oauth/token")) + await asyncio.wait_for(entered.wait(), 5) + assert gate.active == 1 # actual handler ended, outer tail remains counted + assert observed == [1] + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + assert not task.done() + barrier.record_retirement.assert_not_awaited() + if cancel_tail: + task.cancel() + await asyncio.wait_for(canceled.wait(), 5) + assert gate.active == 1 and not task.done() + assert not await barrier.retire_process() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_inner_scope_refuses_when_close_lands_after_outer_admission(): + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + handler, send = AsyncMock(), AsyncMock() + + async def between(scope, receive, send): + assert gate.active == 1 + gate.close() + await ModelMaintenanceMiddleware(handler)(scope, receive, send) + assert gate.active == 1 + + await RetirementMiddleware(between)( + {"type": "http", "path": "/oauth/token", "method": "POST"}, AsyncMock(), send + ) + handler.assert_not_awaited() + assert send.await_args_list[0].args[0]["status"] == 503 + assert gate.active == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + [ + "/oauth/token", + "/oauth/callback", + "/v1/auth/login", + "/v1/scim/Users", + "/v1/admin/models/reset", + ], +) +async def test_canceled_control_dispatch_remains_counted_until_actual_completion(path): + import asyncio + + from server.core import retirement + from server.transport.model_maintenance import ModelMaintenanceMiddleware + from server.transport.retirement import RetirementMiddleware + + gate = retirement.get_gate() + entered, canceled, finish = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def handler(scope, receive, send): + entered.set() + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + assert gate.active == 2 # work could still mint/write here + + task = asyncio.create_task( + RetirementMiddleware(ModelMaintenanceMiddleware(handler))( + {"type": "http", "path": path, "method": "POST"}, AsyncMock(), AsyncMock() + ) + ) + await entered.wait() + task.cancel() + await canceled.wait() + gate.close() + assert gate.active == 2 and not task.done() + finish.set() + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_registered_descendant_blocks_after_outer_and_inner_completed(monkeypatch): + import asyncio + + import httpx + + from server.core import retirement + from server.core.background import create_deferred_task + + gate = retirement.get_gate() + entered, finish = asyncio.Event(), asyncio.Event() + tasks = [] + + async def descendant(): + entered.set() + await finish.wait() + + async def handler(): + tasks.append(create_deferred_task(descendant(), name="retirement-test-descendant")) + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + assert (await client.post("/oauth/token")).status_code == 200 + await entered.wait() + assert gate.active == 0 + gate.install("test") + gate.enrolled = True + gate.stopped() + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + assert not await barrier.retire_process() + barrier.record_retirement.assert_not_awaited() + finish.set() + await asyncio.gather(*tasks) + + +@pytest.mark.asyncio +async def test_certificates_fail_closed_in_original_parsers(): + from pathlib import Path + from types import ModuleType + + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + previous = ModuleType("old_provider_process") + exec(compile(source, "old_provider_process.py", "exec"), previous.__dict__) + proof = provider_process.certified(provider_process.host_proof()) + assert len(proof.split("|")) == 9 + assert not previous.proves_stopped(proof) + assert not await previous.pod_gone(proof) + assert provider_process.has_certificate(proof) + assert not provider_process.has_certificate(proof.replace("cell-retirement-v1", "unknown-v2")) + assert not provider_process.has_certificate("||||||||cell-retirement-v1") + + +@pytest.mark.asyncio +async def test_uncertified_retired_rows_remain_census_blockers_and_unprunable(): + from types import SimpleNamespace + + identity = uuid4() + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": ""} + ch = SimpleNamespace(query_all=AsyncMock(return_value=[row]), execute=AsyncMock()) + assert await store.participants(ch=ch) == [row] + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_prune_rechecks_latest_certificate_and_version(): + from types import SimpleNamespace + + identity = uuid4() + proof = provider_process.certified(provider_process.host_proof()) + row = {"participant_id": identity, "state": "retired", "version": 7, "host_proof": proof} + newer = {**row, "state": "running", "version": 8} + ch = SimpleNamespace(query_all=AsyncMock(side_effect=[[row], [newer]]), execute=AsyncMock()) + assert await store.prune_retired(ch=ch) == 0 + ch.execute.assert_not_awaited() + + +def test_production_topology_installs_outer_and_inner_scopes(): + from server.main import verify_retirement_topology + + verify_retirement_topology() + + +@pytest.mark.parametrize("migrate", [False, True]) +@pytest.mark.asyncio +async def test_server_enrolls_after_pair_verification_before_first_cell_read(monkeypatch, migrate): + from types import SimpleNamespace + + from server import boot, main + from server.auth import jwt + from server.core import object_store + from server.observability import instance + from server.settings import settings + + events = [] + + class FirstRead(Exception): + pass + + async def paired(*args, **kwargs): + events.append("paired") + + async def enrolled(): + assert events[-1] == "paired" + events.append("enrolled") + + async def first_read(): + assert events[-1] == "enrolled" + events.append("read") + raise FirstRead + + async def migrations(*args, enabled, **kwargs): + assert enabled is migrate + return 0 # disabled and no-pending-migrations boots both need enrollment + + monkeypatch.setattr(instance, "require_instance_label", lambda *args, **kwargs: None) + monkeypatch.setattr(jwt, "require_real_signing_secret", lambda **kwargs: None) + monkeypatch.setattr(boot, "require_identity_settings", lambda: None) + monkeypatch.setattr(boot, "require_safe_bind", lambda **kwargs: None) + monkeypatch.setattr(object_store, "require_client", lambda: None) + monkeypatch.setattr(main, "get_client", lambda: SimpleNamespace(connect=AsyncMock())) + monkeypatch.setattr(boot, "require_clickhouse_ready", AsyncMock()) + monkeypatch.setattr(boot, "register_readiness_checks", lambda ch: None) + monkeypatch.setattr(boot, "run_fenced_boot_migrations", migrations) + monkeypatch.setattr(boot, "require_keeper", AsyncMock()) + monkeypatch.setattr(boot, "require_storage_authority", paired) + monkeypatch.setattr(boot, "start_cell_gate", first_read) + monkeypatch.setattr(barrier, "register_process", enrolled) + monkeypatch.setattr(settings, "boot_migrate", migrate) + with pytest.raises(FirstRead): + async with main.lifespan(main.app): + pytest.fail("Only the first read is under test") + assert events == ["paired", "enrolled", "read"] + + +def test_cli_connects_verified_then_enrolls_then_reads_acceptance(monkeypatch): + from contextlib import asynccontextmanager + from types import SimpleNamespace + + from server.cli import _ch + from server.core import pg_profile + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verified(ch): + assert ch is client + events.append("verified") + + @asynccontextmanager + async def command(): + events.append("enrolled") + yield + events.append("completed") + + async def accepting(orgs): + assert events == ["verified", "enrolled"] + events.append("read") + + async def work(ch): + assert events[-1] == "read" + events.append("work") + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(provider_command, "model_command", command) + monkeypatch.setattr(_ch, "check_orgs", accepting) + assert _ch.run_connected(work, orgs=("example",)) == 7 + assert events == ["verified", "enrolled", "read", "work", "completed"] + + +def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census(): + import ast + from pathlib import Path + + root = Path(__file__).parents[2] / "server" + constructors, writers, commands, stopped, plain = set(), set(), set(), set(), set() + for path in root.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + if not isinstance(node, ast.Call): + continue + func = node.func + name = ( + func.id + if isinstance(func, ast.Name) + else func.attr + if isinstance(func, ast.Attribute) + else "" + ) + relative = str(path.relative_to(root)) + if name == "RetirementProof": + constructors.add(relative) + if name == "record_retirement": + writers.add(relative) + if name == "model_command": + commands.add(relative) + if name == "participant_command": + plain.add(relative) + if name == "stopped": + stopped.add(relative) + assert constructors == {"core/retirement.py"} + assert writers == {"insights/app/provider_barrier.py"} + assert commands == { + "cli/_ch.py", + "cli/code_run.py", + "cli/import_clickgap.py", + "cli/import_recall.py", + } + assert plain == { + "cli/sql_access.py", + "cli/analytics_export.py", + "cli/maintenance.py", + "insights/app/provider_command.py", + } + assert stopped == {"main.py", "worker_main.py", "insights/app/provider_command.py"} + + +def test_manual_retirement_has_no_affirmation_bypass(): + import argparse + + from server.cli.participant import register_participant_cli + + parser = argparse.ArgumentParser() + register_participant_cli(parser.add_subparsers(dest="command", required=True)) + with pytest.raises(SystemExit) as error: + parser.parse_args(["participant", "retire", "--participant", str(uuid4()), "--yes"]) + assert error.value.code == 2 + + +def test_fork_cannot_inherit_admission(monkeypatch): + from server.core import retirement + + gate = retirement.get_gate() + gate.install("server") + gate.enrolled = True + monkeypatch.setattr(retirement, "_pid", -1) + child = retirement.get_gate() + assert child is not gate and child.role == "server" and not child.enrolled + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_state", ["404", "replacement", "Failed", "Succeeded", "unreachable"]) +async def test_pod_object_death_never_retires_work_that_can_resume( + monkeypatch, tmp_path, api_state +): + import asyncio + from pathlib import Path + from types import ModuleType + + import httpx + + from server.core import retirement + + old = ModuleType("old_reaper") + source = ( + Path(__file__).parents[1] / "fixtures" / "provider_process_before_retirement.py" + ).read_text() + exec(compile(source, "old_reaper.py", "exec"), old.__dict__) + uid = str(uuid4()) + proof = "|".join( + ("partitioned-host", "boot", uid + "/loom", "4026", "7", "99", "loom", "old-pod") + ) + (tmp_path / "token").write_text("test-token") + monkeypatch.setattr(old, "_SA", str(tmp_path)) + monkeypatch.setenv("POD_NAMESPACE", "loom") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "kubernetes.invalid") + seen = [] + + def answer(request): + seen.append(request) + if api_state == "unreachable": + raise httpx.ConnectError("partitioned", request=request) + return httpx.Response( + 404 if api_state == "404" else 200, + json={ + "metadata": {"uid": str(uuid4()) if api_state == "replacement" else uid}, + "status": {"phase": api_state}, + }, + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + httpx, "AsyncClient", lambda **kw: real_client(transport=httpx.MockTransport(answer)) + ) + # Demonstrate the exact old mistake before testing its replacement. + assert await old.pod_gone(proof) is (api_state != "unreachable") + participant = uuid4() + row = { + "participant_id": participant, + "state": "running", + "version": 9, + "expired": True, + "host_proof": proof, + } + monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=[row])) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + monkeypatch.setattr(barrier, "_STALE", None) + resume, admitted = asyncio.Event(), asyncio.Event() + effects = [] + + async def paused(): + async with retirement.get_gate().admit(): + admitted.set() + await resume.wait() + effects.append("mint after object deletion") + + task = asyncio.create_task(paused()) + await admitted.wait() + await barrier.sweep_stopped(now=5000) + barrier.record_retirement.assert_not_awaited() + assert barrier.stale_participants()[0]["participant_id"] == str(participant) + assert len(seen) == 1 # the current reaper never asks the API for a death certificate + resume.set() + await task + assert effects == ["mint after object deletion"] + + +@pytest.mark.asyncio +async def test_production_child_task_cancellation_stays_counted_until_completion(): + import asyncio + import contextlib + + import anyio + import httpx + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + entered.set() + with anyio.CancelScope(shield=True): + await finish.wait() + return {"ok": True} + + app = _request_app(handler) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await entered.wait() + task.cancel() + # Let BaseHTTPMiddleware propagate cancellation to its downstream child. + for _ in range(5): + await asyncio.sleep(0) + gate.close() + assert gate.active > 0 + assert not task.done() + finish.set() + with contextlib.suppress(asyncio.CancelledError): + await task + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_shutdown_budget_does_not_wait_for_cancellation_deferring_drains(monkeypatch): + import asyncio + import time + + from server import boot, main + from server.core import instances, retirement + + gate = retirement.get_gate() + gate.install("test") + gate.enrolled = True + gate.stopped() + canceled, finish = asyncio.Event(), asyncio.Event() + owned = [] + + async def drains(*args): + owned.append(asyncio.current_task()) + try: + await finish.wait() + except asyncio.CancelledError: + canceled.set() + await finish.wait() + + async def leave(*args, **kwargs): + assert not await barrier.retire_process() + + monkeypatch.setattr(instances, "stop", AsyncMock()) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 0.01) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + started = time.monotonic() + await asyncio.wait_for(main._shut_down(None, None, None, None), timeout=0.5) + assert time.monotonic() - started < 0.5 + await canceled.wait() + assert not owned[0].done() + barrier.record_retirement.assert_not_awaited() + finish.set() + await owned[0] + + +def test_nonmodel_principal_command_enrolls_before_acceptance(monkeypatch): + from types import SimpleNamespace + + from server.cli import _ch, sql_access + from server.core import pg_profile, retirement + from server.insights.app import provider_command + + events = [] + client = SimpleNamespace(connected=True, disconnect=AsyncMock()) + + async def verify(ch): + events.append("verified") + + async def register(): + assert events == ["verified"] + retirement.get_gate().enrolled = True + events.append("registered") + + async def acceptance(orgs): + assert events == ["verified", "registered"] + events.append("acceptance") + + async def work(ch): + assert events[-1] == "acceptance" + assert retirement.get_gate().active == 1 + return 7 + + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(provider_command, "register_process", register) + monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + assert sql_access._connected(work, orgs=("example",)) == 7 + assert retirement.get_gate().closed and retirement.get_gate().active == 0 + + +@pytest.mark.asyncio +async def test_stream_and_response_background_cleanup_remain_counted(monkeypatch): + import asyncio + + import httpx + from starlette.background import BackgroundTask + from starlette.responses import StreamingResponse + + from server.core import retirement + + gate = retirement.get_gate() + body_entered, body_finish = asyncio.Event(), asyncio.Event() + cleanup_entered, cleanup_finish = asyncio.Event(), asyncio.Event() + + async def body(): + yield b"first" + body_entered.set() + await body_finish.wait() + yield b"last" + + async def cleanup(): + cleanup_entered.set() + await cleanup_finish.wait() + + async def handler(): + return StreamingResponse(body(), background=BackgroundTask(cleanup)) + + app = _request_app(handler) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + task = asyncio.create_task(client.post("/oauth/token")) + await body_entered.wait() + assert gate.active == 2 + gate.install("test") + gate.enrolled = True + gate.stopped() + assert not await barrier.retire_process() + body_finish.set() + await cleanup_entered.wait() + assert gate.active == 2 and not task.done() + assert not await barrier.retire_process() + cleanup_finish.set() + assert (await task).content == b"firstlast" + assert gate.active == 0 + + +@pytest.mark.asyncio +async def test_websocket_lifetime_and_shutdown_refusal_share_the_same_counter(): + import asyncio + + from server.core import retirement + + entered, finish = asyncio.Event(), asyncio.Event() + gate = retirement.get_gate() + + async def handler(): + return {} + + from starlette.websockets import WebSocket + + async def websocket(ws: WebSocket): + await ws.accept() + entered.set() + await finish.wait() + await ws.close() + + app = _request_app(handler) + app.add_api_websocket_route("/socket", websocket) + scope = { + "type": "websocket", + "path": "/socket", + "scheme": "ws", + "query_string": b"", + "headers": [(b"host", b"localhost")], + "subprotocols": [], + "client": ("127.0.0.1", 1234), + "server": ("localhost", 80), + } + task = asyncio.create_task( + app(scope, AsyncMock(return_value={"type": "websocket.connect"}), AsyncMock()) + ) + await entered.wait() + assert gate.active == 2 + gate.close() + send = AsyncMock() + await app(scope, AsyncMock(), send) + send.assert_awaited_once_with({"type": "websocket.close", "code": 1013}) + assert gate.active == 2 + finish.set() + await task + assert gate.active == 0 diff --git a/tests/unit/test_provider_barrier.py b/tests/unit/test_provider_barrier.py index 8cac6eceb8..551a8c998f 100644 --- a/tests/unit/test_provider_barrier.py +++ b/tests/unit/test_provider_barrier.py @@ -29,6 +29,11 @@ def _row(pid, transition=None, state="running", *, version=1, expired=False): @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().install("test") + retirement.get_gate().enrolled = True + retirement.get_gate().producers_stopped = True identity, transition = uuid4(), uuid4() gate = ModelWorkGate() monkeypatch.setattr(barrier, "get_model_work_gate", lambda: gate) @@ -37,6 +42,7 @@ def setup(monkeypatch): monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) record = AsyncMock() monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", AsyncMock()) monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) rows = [_row(identity, transition, "drained")] monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) @@ -188,7 +194,7 @@ def test_a_heartbeat_carries_the_latest_state_above_any_version_seen(setup): "state": "drained", "version": 10**20 + 1, "stale_after_s": barrier.STALE_AFTER_S, - "host_proof": barrier.host_proof(), + "host_proof": barrier.participant_proof(), } # An acknowledgement stamped after the beat outranks it. assert barrier._OWN.stamp(transition, "drained")[2] > row[3] @@ -205,12 +211,14 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m await barrier.register_process() assert beat.running and len(ch.inserts) == 1 tags: list[str] = [] - record.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + barrier.record_retirement.side_effect = lambda *_a, **_k: tags.append(current_query_tag()) + beat.signal_stop() + beat.thread.join(timeout=5) assert await barrier.retire_process() assert beat.stopping.is_set() # signalled, not joined: the thread ends after its beat beat.thread.join(timeout=5) assert not beat.running - assert record.call_args.args[2] == "retired" + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() assert tags == ["provider.participant_heartbeat"] and current_query_tag() == "" beat.ch = ch beat.beat() @@ -218,7 +226,7 @@ async def test_register_writes_the_first_row_and_retire_stops_the_beats(setup, m @pytest.mark.asyncio -async def test_a_boot_that_fails_after_enrolling_retires_its_row(setup, monkeypatch): +async def test_a_boot_that_fails_does_not_certify_a_pending_heartbeat(setup, monkeypatch): _, _, _, record = setup monkeypatch.setattr(barrier, "_HEARTBEAT", barrier._Heartbeat(lambda: _SyncCH())) monkeypatch.setattr(barrier, "_registered", False) @@ -232,7 +240,10 @@ async def lifespan(_app): with pytest.raises(RuntimeError, match="migrations failed"): async with barrier.participant_lifespan(lifespan)(None): pytest.fail("a failed boot never serves") - assert record.call_args.args[2] == "retired" + barrier.record_retirement.assert_not_awaited() + barrier._HEARTBEAT.thread.join(timeout=5) + assert await barrier.retire_process() + assert barrier.record_retirement.call_args.args[0].participant == barrier.process_identity() @pytest.mark.asyncio @@ -263,8 +274,9 @@ async def test_retire_participant_refuses_a_live_one(setup, monkeypatch): assert await barrier.retire_participant(other) == "live" record.assert_not_awaited() monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) - assert await barrier.retire_participant(other) == "retired" - record.assert_awaited_once_with(other, UUID(int=0), "retired", after=5, ch=None) + assert await barrier.retire_participant(other) == "live" + record.assert_not_awaited() + barrier.record_retirement.assert_not_awaited() def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): @@ -281,7 +293,7 @@ def test_loom_participant_retire_refuses_a_live_one(setup, monkeypatch, capsys): monkeypatch.setattr(_ch, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = argparse.Namespace(participant=str(other), yes=True) assert participant._cmd_retire(args) == 1 - assert "still running" in capsys.readouterr().err + assert "no verifiable exact physical-death proof" in capsys.readouterr().err record.assert_not_awaited() @@ -434,14 +446,13 @@ async def _hang(*_a, **_k): @pytest.mark.asyncio -async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(): +async def test_a_live_row_expires_after_the_heartbeat_window_and_a_retired_one_at_once(monkeypatch): ch = SimpleNamespace(insert=AsyncMock()) - await pp.record_participant(uuid4(), UUID(int=0), "running", ch=ch) + await pp.record_participant(pp.process_identity(), UUID(int=0), "running", ch=ch) (row,) = ch.insert.call_args.args[1] assert row["stale_after_s"] == pp.STALE_AFTER_S > barrier.HEARTBEAT_S - await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) - (row,) = ch.insert.call_args.args[1] - assert row["stale_after_s"] == 0 + with pytest.raises(ValueError, match="state"): + await pp.record_participant(uuid4(), UUID(int=0), "retired", ch=ch) @pytest.mark.asyncio diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 347506b522..c6ba785ab4 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -10,6 +10,9 @@ @pytest.fixture def setup(monkeypatch): + from server.core import retirement + + retirement.get_gate().enrolled = True gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 60cead66a5..d1ee7960e3 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -70,7 +70,7 @@ def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) - assert pp.proves_stopped(_with(container="c" * 64, pidns="1")) + assert not pp.proves_stopped(_with(container="c" * 64, pidns="1")) @pytest.mark.parametrize("theirs", ["d" * 64, ""]) @@ -152,127 +152,11 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) ) record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) + monkeypatch.setattr(barrier, "record_retirement", record) monkeypatch.setattr(barrier, "_last_here", -float("inf")) assert await barrier.retire_dead_here(now=100.0) == 1 - record.assert_awaited_once_with(dead, UUID(int=0), "retired", after=5) + record.assert_awaited_once() + proof = record.await_args.args[0] + assert proof.participant == dead and proof.after == 5 + assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled - - -# -- a pod Kubernetes deleted ------------------------------------------------ - -_POD_UID = "0b6f2c1e-1111-2222-3333-444455556666" - - -def _pod_proof( - namespace: str = "loom", container: str = f"{_POD_UID}/loom", pod: str = "loom-7d9-abc" -) -> str: - # The host name is the pod name cut to 63 characters; the lookup uses the pod name. - return "|".join((pod[:63], "boot", container, "4026", "7", "99", namespace, pod)) - - -@pytest.fixture -def kube(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> Any: - """A service-account mount, the API's address and a stand-in API: it answers - the (status, body) pairs queued in `answers` and keeps what it saw in `seen`.""" - from types import SimpleNamespace - - import httpx - - (tmp_path / "token").write_text("tok") - (tmp_path / "ca.crt").write_text("") - monkeypatch.setattr(pp, "_SA", str(tmp_path)) - monkeypatch.setenv("POD_NAMESPACE", "loom") - monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.96.0.1") - api = SimpleNamespace(answers=[], seen=[]) - - def handler(request: httpx.Request) -> httpx.Response: - api.seen.append(request) - status, body = api.answers.pop(0) - return httpx.Response(status, json=body) - - real = httpx.AsyncClient - monkeypatch.setattr( - httpx, - "AsyncClient", - lambda **kw: real(transport=httpx.MockTransport(handler), timeout=kw.get("timeout")), - ) - return api - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "status,uid,phase,gone", - [ - (404, "", "", True), - (200, "another-uid", "Running", True), - (200, _POD_UID, "Failed", True), # evicted: the object stays, its containers stopped - (200, _POD_UID, "Succeeded", True), - (200, _POD_UID, "Running", False), - (200, _POD_UID, "Pending", False), - (403, "", "", False), - ], -) -async def test_a_pod_is_gone_only_when_the_api_says_so(kube, status, uid, phase, gone) -> None: - kube.answers.append((status, {"metadata": {"uid": uid}, "status": {"phase": phase}})) - assert await pp.pod_gone(_pod_proof()) is gone - assert kube.seen[0].url.path == "/api/v1/namespaces/loom/pods/loom-7d9-abc" - assert kube.seen[0].headers["authorization"] == "Bearer tok" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "proof", - [ - _pod_proof(namespace="other"), - _pod_proof(container="a" * 64), - _pod_proof(pod=""), - "h|b|c|p|1|2", - "h|b|" + _POD_UID + "/loom|p|1|2|loom", # a 7-part proof (no pod name) proves nothing - "", - ], -) -async def test_a_pod_elsewhere_or_outside_kubernetes_is_never_gone(kube, proof) -> None: - assert await pp.pod_gone(proof) is False - assert kube.seen == [] # nothing to ask: another namespace, a Docker container, an old row - - -@pytest.mark.asyncio -async def test_the_leader_retires_an_expired_participant_whose_pod_is_gone( - monkeypatch: pytest.MonkeyPatch, -) -> None: - me, lost, busy, kept = (uuid4() for _ in range(4)) - monkeypatch.setattr(barrier, "process_identity", lambda: me) - rows = [ - _row(me, pp.host_proof()), - _row(lost, _pod_proof(), expired=True, version=9), - _row(busy, _pod_proof(), expired=True), - _row(kept, _pod_proof(), expired=True), - ] - monkeypatch.setattr(barrier, "participants", AsyncMock(return_value=rows)) - monkeypatch.setattr(barrier, "prune_retired", AsyncMock()) - monkeypatch.setattr( - barrier, "running_queries", AsyncMock(side_effect=lambda p: int(str(p) == str(busy))) - ) - asked: list[str] = [] - - async def gone(proof: str) -> bool: - asked.append(proof) - return len(asked) == 1 # the first asked (lost) is gone, kept's pod still exists - - monkeypatch.setattr(barrier, "pod_gone", gone) - record = AsyncMock() - monkeypatch.setattr(barrier, "record_participant", record) - monkeypatch.setattr(barrier, "_STALE", None) - await barrier.sweep_stopped(now=5000.0) - record.assert_awaited_once_with(lost, UUID(int=0), "retired", after=9) - assert len(asked) == 2 # a participant with a query running is never asked about - assert [p["participant_id"] for p in barrier.stale_participants()] == [str(busy), str(kept)] - - -@pytest.mark.asyncio -async def test_a_long_pod_name_is_looked_up_whole_not_by_its_cut_host_name(kube) -> None: - long = "loom-memory-platform-production-workers-7d9c8b6f5d-x4kqz-abcdefgh" - kube.answers.append((200, {"metadata": {"uid": _POD_UID}, "status": {"phase": "Running"}})) - assert await pp.pod_gone(_pod_proof(pod=long)) is False - assert kube.seen[0].url.path == f"/api/v1/namespaces/loom/pods/{long}" diff --git a/tests/unit/test_reextract.py b/tests/unit/test_reextract.py index 852f94d1b0..addaef8003 100644 --- a/tests/unit/test_reextract.py +++ b/tests/unit/test_reextract.py @@ -159,7 +159,9 @@ def test_cli_refuses_the_source_as_target(capsys) -> None: assert "must differ" in capsys.readouterr().err -def test_reextract_token_falls_back_to_the_saved_token(tmp_path, monkeypatch) -> None: +def test_reextract_token_falls_back_to_the_saved_token( + tmp_path, monkeypatch, stub_model_command +) -> None: """--token, else LOOM_API_KEY, else ~/.loom/token: the order every client command uses.""" import argparse diff --git a/tests/unit/test_revocation_web_refresh.py b/tests/unit/test_revocation_web_refresh.py index bbc0f56a5a..8dac8ed351 100644 --- a/tests/unit/test_revocation_web_refresh.py +++ b/tests/unit/test_revocation_web_refresh.py @@ -133,16 +133,28 @@ def _poll_loop(period: float) -> asyncio.Task[None]: t.cancel() -def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown() -> None: - src = (ROOT / "server" / "main.py").read_text() - lifespan = src[src.index("async def lifespan(") :] - assert "auth_loops = await boot.start_token_state()" in lifespan - assert ( - "for loop in (*cell_loops, *auth_loops, overrides_loop):\n loop.cancel()" - in lifespan +@pytest.mark.asyncio +async def test_lifespan_starts_the_token_state_and_cancels_it_at_shutdown(monkeypatch) -> None: + import ast + + from server import boot, main + from server.code.app.index import mirrors + from server.core import retirement + from server.memory.app import embedder_calibration + + tree = ast.parse((ROOT / "server" / "main.py").read_text()) + lifespan = next( + n for n in ast.walk(tree) if isinstance(n, ast.AsyncFunctionDef) and n.name == "lifespan" ) - # the decorator still sits on lifespan, not on the helper above it - assert "@asynccontextmanager\nasync def lifespan(" in src + calls = {ast.unparse(n.func) for n in ast.walk(lifespan) if isinstance(n, ast.Call)} + assert {"boot.start_token_state", "_stop_background"} <= calls + monkeypatch.setattr(mirrors, "stop_warmup", lambda: None) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: None) + monkeypatch.setattr(boot, "cancel_boot_tasks", lambda: None) + task = retirement.get_gate().track(asyncio.create_task(asyncio.Event().wait())) + await main._stop_background(None) + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and retirement.get_gate().closed @pytest.mark.asyncio diff --git a/tests/unit/test_shutdown_deadline.py b/tests/unit/test_shutdown_deadline.py index e8d53b92f2..8f60ab0419 100644 --- a/tests/unit/test_shutdown_deadline.py +++ b/tests/unit/test_shutdown_deadline.py @@ -125,7 +125,7 @@ def close(self): pass # The parent measures from here to the process's exit, imports left out # (CLOCK_MONOTONIC is one clock for every process on the host). print("retiring", time.monotonic(), flush=True) -assert asyncio.run(barrier.retire_process()) +assert not asyncio.run(barrier.retire_process()) """ From 5db501c8e7893f0cccb217011abe4ee1068b402e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 16 Nov 2026 22:52:01 +0000 Subject: [PATCH 093/181] Hold native principal work through command retirement Keep claim lookup and release in one enrolled async body. Run token readiness, keyring loading, signing and audit before the sole command drain and retirement. Refuse native offline minting when paired-store verification or enrollment fails; a keyring-only read failure inside enrolled work can still use the root key. Remove unreachable registration-row branches and duplicate worker task tracking, and name the changed-namespace death regression after its fail-closed contract. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 11 + docs/guides/tokens-and-grants.md | 7 +- docs/reference/cli.md | 7 +- server/auth/keyring.py | 23 +- server/cli/_ch.py | 4 +- server/cli/access.py | 153 +++++++------ .../insights/store/provider_participants.py | 6 +- server/worker_main.py | 2 - tests/unit/snapshots/cli_help.json | 2 +- tests/unit/snapshots/cli_tree.json | 2 +- tests/unit/test_cell_cli_orgs.py | 21 ++ tests/unit/test_cli_hygiene.py | 14 +- tests/unit/test_native_command_retirement.py | 201 ++++++++++++++++++ tests/unit/test_platform_admin_bots.py | 20 +- tests/unit/test_platform_admin_rollout.py | 8 +- tests/unit/test_provider_host_proof.py | 2 +- tests/unit/test_revoke_issued_before.py | 9 +- 17 files changed, 395 insertions(+), 97 deletions(-) create mode 100644 tests/unit/test_native_command_retirement.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 9cd44ad6f4..7fccee817d 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -25,6 +25,17 @@ enrolls before its first fresh primary acceptance read and uses permanent guarde retirement. A serving cell heartbeat includes its `participant_id` beside the fingerprint, allowing a replacement to be joined to the participant registry. Workers children enroll independently and do not publish serving instance rows. +Native commands hold one admission scope through their complete org or principal +work and drain: claim lookup and release share that scope; token readiness, keyring +read, signing and audit share another. Connections and helper calls do not retire +a command that still has principal work to perform. Store pairing is verified +before enrollment and fresh acceptance. Org-free diagnostics do not install an +accepting view. The pre-schema `ch-setup` ownership probe discards its diagnostic +mode result and checks only store/cell identity before raw database user/profile +setup; its optional analyst-row work opens a separate verified, enrolled scope +and obtains fresh acceptance. That bootstrap exception admits no tenant or token +work. + The capability identifies installed code; it does not prove that every older process or unsafe retirement client has disappeared. diff --git a/docs/guides/tokens-and-grants.md b/docs/guides/tokens-and-grants.md index a5bf21c44a..63c919f0d7 100644 --- a/docs/guides/tokens-and-grants.md +++ b/docs/guides/tokens-and-grants.md @@ -26,7 +26,12 @@ loom mint-token --subject my-bot --org acme --namespaces support --roles writer `loom` ships with the server package and the Docker image, not with the SDK; in a container run `docker exec loom mint-token ...`. Minting needs the signing key, -`LOOM_JWT_SECRET`. A fresh clone gets a development secret from `.env.example`. +`LOOM_JWT_SECRET`, and access to the deployment's migrated stores. The command verifies +the paired stores and enrolls before reading the accepting cell state or signing. +Offline native minting and minting before migrations refuse. If only the keyring read +fails after successful enrollment, an ordinary mint can still sign with the root key. +Platform-admin minting also requires a fresh readable fleet and a recorded audit event. A fresh clone gets a development +secret from `.env.example`. `loom mint-token` and `loom doctor` warn while it is in use, and a deployment with auth on refuses to start with it (except the `make dev` stack). diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3e0b9068ea..aa4218fbde 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -38,7 +38,10 @@ Commands fall into two kinds: whatever `--token` says, `loom init` refuses before it signs anything up, and `loom call` exchanges the workload token itself. -`loom mint-token` needs only `LOOM_JWT_SECRET`: it signs a token locally. +`loom mint-token` signs locally with `LOOM_JWT_SECRET` after verifying its paired +stores and enrolling as a participant. It refuses when those stores cannot be verified +or enrollment fails, including offline and before the deployment is migrated. Run it +against the deployment's migrated stores. **Session commands** (`login`, `logout`, `whoami`, `call`, `token`) are the Python SDK's own (`loom.cli`; `python -m loom …` with only the SDK installed), so `loom` @@ -304,7 +307,7 @@ python scripts/codegen/docs_reference.py | `loom import-recall` | Import an external vector-recall agent-memory SQLite export into Loom. | `--sqlite`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom import-brain` | Import a markdown agent 'brain' directory into Loom: MEMORY.md (always-injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on-demand WARM layer) → knowledge.add docs. | `--dir`, `--org`, `--namespace`, `--agent`, `--base-url`, `--token`, `--dry-run` | | `loom claim-release` | Release a self-signup claim: revoke its grants, clear the owner, deactivate it. | `--org`, `--namespace`, `-y` | -| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | +| `loom mint-token` | Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). | `--subject`, `--org`, `--namespaces`, `--roles`, `--platform-admin`, `--namespace-role`, `--ttl-days`, `--save`, `--read-scope`, `--read-user-id`, `--traffic-class`, `--consumer`, `--service-account`, `--user-token`, `--json` | | `loom revoke-token` | Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). | `--jti`, `--subject`, `--list`, `--issued-before`, `--scim`, `--reason`, `--expires-at`, `--json` | | `loom sso` | Carry password accounts over to single sign-on. | | | `loom sso adopt` | Let SSO sign-ins reach the password accounts whose email is at --domain: each person then signs in with the identity provider to their own account, its namespaces and data. | `--domain`, `--org`, `--skip`, `--keep-tokens`, `--apply`, `--dry-run` | diff --git a/server/auth/keyring.py b/server/auth/keyring.py index d4624a4369..ff5c413d4b 100644 --- a/server/auth/keyring.py +++ b/server/auth/keyring.py @@ -162,8 +162,9 @@ def sign_with_root_only() -> None: def load_for_offline_mint(*, run: Callable[..., Any] | None = None) -> bool: """For a process that mints without serving (a CLI, a dev or bench script): read the keyring once, or sign as kid `default` when ClickHouse - cannot be read. `run` is the CLI's `run_connected`; without it the read - uses a client of its own. True when it was read. Raises ValueError when + cannot be read. `run` can supply a connected read; without it the read + uses a client of its own. Native CLI minting uses `load_for_connected_mint` + inside its enrolled command instead. True when it was read. Raises ValueError when the keyring was recorded under another root secret: no server would accept what this process signs; through `run`, raises AuthorityMismatch when the shell is configured for another deployment (`connect_checked`).""" @@ -191,6 +192,24 @@ async def _read() -> bool: raise # a shell configured for another deployment: refused, not signed by root except Exception: # noqa: BLE001 — an offline mint signs with the root secret read = False + return _offline_mint_result(read) + + +async def load_for_connected_mint(*, ch: Any) -> bool: + """Read signing keys inside an already verified, enrolled native command.""" + from server.core.pg import AuthorityMismatch + from server.core.retirement import ProcessStopping + + try: + read = await refresh(ch=ch) + except (AuthorityMismatch, ProcessStopping): + raise + except Exception: # noqa: BLE001 — an enrolled mint can sign with the root secret + read = False + return _offline_mint_result(read) + + +def _offline_mint_result(read: bool) -> bool: if not read: print(" · signing keyring not read; signing as kid 'default'", file=sys.stderr) sign_with_root_only() diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 184aa1f6eb..c67479b965 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -193,7 +193,9 @@ async def check_setup_stores() -> None: gated connect (`loom ch-setup`), the gate a gated connect would run, before it writes: its ClickHouse, where it already records a deployment, must record its Postgres's, and the Postgres `first` row must not name another cell. A ClickHouse with no Loom - database or record yet (a service set up before the first boot) passes.""" + database or record yet (a service set up before the first boot) passes. This + diagnostic discards the mode result and never installs an accepting view; + subsequent Loom row work must verify, enroll and check acceptance afresh.""" from server.core import pg, pg_profile from server.core.ch import get_client from server.core.ch_errors import error_code, error_line diff --git a/server/cli/access.py b/server/cli/access.py index 25cfc05667..482aea7b1a 100644 --- a/server/cli/access.py +++ b/server/cli/access.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +import asyncio import json import sys from datetime import UTC @@ -39,31 +40,34 @@ async def _lookup_owner(ch: Any) -> str | None: return None return str(row.get("owner_email") or "") or "(unknown)" - owner = run_connected(_lookup_owner, org_free=reads("reads")) - if owner is None: - print( - f"no claim found for {args.org}/{args.namespace}", - file=sys.stderr, - ) - return 1 - if not args.yes: - print( - f"This will release the claim on {args.org}/{args.namespace}" - f" (currently owned by {owner!r})." - ) - print( - "Grants are revoked and the namespace deactivated. Its memories are " - "kept, and the name stays reserved: nobody can claim it again." - ) - try: - resp = input("Continue? [y/N] ").strip().lower() - except EOFError: # no terminal to answer (a pipe, a cron): the answer is no - resp = "" - if resp not in ("y", "yes"): - print("aborted") - return 1 - async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + owner = await _lookup_owner(ch) + if owner is None: + print( + f"no claim found for {args.org}/{args.namespace}", + file=sys.stderr, + ) + return 1 + if not args.yes: + print( + f"This will release the claim on {args.org}/{args.namespace}" + f" (currently owned by {owner!r})." + ) + print( + "Grants are revoked and the namespace deactivated. Its memories are " + "kept, and the name stays reserved: nobody can claim it again." + ) + try: + resp = (await asyncio.to_thread(input, "Continue? [y/N] ")).strip().lower() + except EOFError: # no terminal to answer (a pipe, a cron): the answer is no + resp = "" + if resp not in ("y", "yes"): + print("aborted") + return 1 + # Confirmation can wait while the cell changes; check again before mutation. + await check_acceptance(orgs=(args.org,)) out = await release_claim( org=args.org, namespace=args.namespace, @@ -110,8 +114,6 @@ def _invalid_namespace_names(names: list[str]) -> list[str]: def _cmd_mint_token(args: argparse.Namespace) -> int: from server.auth import keyring - from server.auth.authz import GRANT_ROLES - from server.auth.jwt import mint_token, new_jti from server.core.principals import RESERVED_SUBJECT, is_reserved_subject if is_reserved_subject(args.subject): @@ -122,19 +124,49 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: if refused: print(f" ✗ {refused}", file=sys.stderr) return 2 - if args.platform_admin and not _platform_admin_rollout_ready(): + + async def _go(ch: Any) -> int: + from server.cli._ch import check_acceptance + + if args.platform_admin and not await _platform_admin_rollout_ready(ch): + print( + " ✗ platform-admin tokens require a readable fleet with the rollout complete", + file=sys.stderr, + ) + return 1 + try: + await keyring.load_for_connected_mint(ch=ch) + except ValueError as e: # another root: no server would accept the token + print(f" ✗ {e}", file=sys.stderr) + return 1 + # Keyring reads can wait; signing must use a current accepting view too. + await check_acceptance(orgs=(args.org,)) + minted = _mint_token(args) + if isinstance(minted, int): + return minted + token, summary = minted + if args.platform_admin and not await _audit_platform_admin_mint(args, summary): + print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) + return 1 + return _emit_minted_token(args, token, summary) + + try: + return run_connected(_go, orgs=(args.org,)) + except AuthorityMismatch: + raise + except Exception as e: # noqa: BLE001 — a mint without verified enrollment is refused print( - " ✗ platform-admin tokens require a readable fleet with the rollout complete", + f" ✗ token mint did not complete in its enrolled scope: {type(e).__name__}", file=sys.stderr, ) return 1 - try: - keyring.load_for_offline_mint( - run=lambda work: run_connected(work, org_free=reads("reads the signing keyring")) - ) - except ValueError as e: # another root: no server would accept the token - print(f" ✗ {e}", file=sys.stderr) - return 1 + + +def _mint_token(args: argparse.Namespace) -> tuple[str, dict[str, Any]] | int: + """Build and sign only inside the command's enrolled admission scope.""" + from server.auth.authz import GRANT_ROLES + from server.auth.jwt import mint_token, new_jti + ttl_seconds = (args.ttl_days * 86400) if args.ttl_days else None # Every minted token carries a jti; generated here so # the summary can name it beside (or instead of) the token — it is the @@ -238,11 +270,7 @@ def _cmd_mint_token(args: argparse.Namespace) -> int: "consumer": args.consumer or "", } - if args.platform_admin and not _audit_platform_admin_mint(args, summary): - # Never handed out unrecorded: the role spans every org. - print(" ✗ the mint could not be recorded in the audit log; no token", file=sys.stderr) - return 1 - return _emit_minted_token(args, token, summary) + return token, summary # not a setting: the longest a minted platform-admin token may live; longer is a bot's job @@ -272,45 +300,40 @@ def _platform_admin_refusal(args: argparse.Namespace) -> str | None: return None -def _platform_admin_rollout_ready() -> bool: +async def _platform_admin_rollout_ready(ch: Any) -> bool: from server.auth import machines try: - return bool( - run_connected( - machines.platform_admin_tokens_ready, org_free=reads("reads the live instances") - ) - ) + return bool(await machines.platform_admin_tokens_ready(ch)) except AuthorityMismatch: - raise # a shell configured for another deployment: its own one-line refusal + raise except Exception as e: # noqa: BLE001 — an unread fleet refuses an admin token print(f" ✗ fleet state could not be read: {type(e).__name__}", file=sys.stderr) return False -def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: +async def _audit_platform_admin_mint(args: argparse.Namespace, summary: dict[str, Any]) -> bool: """Record a platform-admin mint before it is handed out; False when the audit row could not be written.""" import getpass from server.observability import audit - async def _emit(ch: Any) -> str: - return await audit.emit( - "auth.platform_admin_minted", - org=args.org, - namespace="-", - actor=f"cli:{getpass.getuser()}", - subject={ - "subject": args.subject, - "jti": summary["jti"], - "namespaces": summary["namespaces"], - "ttl_days": args.ttl_days, - }, - ) - try: - return bool(run_connected(_emit, orgs=(args.org,))) + return bool( + await audit.emit( + "auth.platform_admin_minted", + org=args.org, + namespace="-", + actor=f"cli:{getpass.getuser()}", + subject={ + "subject": args.subject, + "jti": summary["jti"], + "namespaces": summary["namespaces"], + "ttl_days": args.ttl_days, + }, + ) + ) except Exception as e: # noqa: BLE001 — an unrecorded mint is refused, never handed out print(f" ✗ audit log not reached: {type(e).__name__}: {e}", file=sys.stderr) return False @@ -714,7 +737,9 @@ def _add_revoke_token_parser(sub: Any) -> None: def _add_mint_token_parser(sub: Any) -> None: """The `mint-token` subcommand.""" - p_mint = sub.add_parser("mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET).") + p_mint = sub.add_parser( + "mint-token", help="Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores)." + ) p_mint.add_argument("--subject", required=True) p_mint.add_argument("--org", required=True) p_mint.add_argument( diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index da04df218d..faa8bc1d54 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -111,10 +111,8 @@ async def record_participant( "transition_id": transition, "state": state, "version": version if version is not None else max(time_ns(), after + 1), - # A retired row is stopped at once, so every build's `provider-lock clear` - # frees its holder without waiting out the heartbeat window. - "stale_after_s": 0 if state == "retired" else STALE_AFTER_S, - "host_proof": participant_proof() if own else "", + "stale_after_s": STALE_AFTER_S, + "host_proof": participant_proof(), } ], ) diff --git a/server/worker_main.py b/server/worker_main.py index c265c7c124..4a606a4291 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -63,8 +63,6 @@ async def _trackers() -> None: track(boot.start_runtime_overrides()), ] await boot.await_warm_up(settings.boot_warmup_timeout_s, ("runtime_overrides",)) - for task in tasks: - retirement.get_gate().track(task) return tasks diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 37e0e5e39e..f0f3075ad1 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -2,7 +2,7 @@ "loom": [ "usage: loom [-h] [--version] {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} ...", "Loom — vendor-neutral agent memory plane on ClickHouse.", - "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", + "positional arguments: {code,decisions,knowledge,export-okf,import-okf,okf-lint,up,down,ch-setup,participant,deployment,settings,secrets,signing-key,provider-lock,export-analytics,sql-access,reembed,reextract,deploy-delta,entity-export,audit-namespace,mentor,serve,migrate,reconcile-status,period-digest,demote-activity-narration,recalibrate,import-clickgap,import-recall,import-brain,claim-release,mint-token,revoke-token,sso,bot,doctor,report,drops,namespaces-gc,erasure-re-verify,reconcile,router,cell,tracker-probe,tracker-contract-check,tracker-dupcheck-fit,tracker-embed-reset,namespaces-adopt,sensitivity-sweep,integrity-repair,rechunk,export-actions,lexical-stats-rebuild,version,install,demo,init,ingest-transcript,memory-search,set-host,login,logout,whoami,call,token} code Code intelligence plane operations. decisions Decision records mined from documents. knowledge Knowledge documents and their sources. export-okf Render a namespace as an OKF v0.2 bundle (Markdown + frontmatter) — the git-reviewable projection of Loom memory. import-okf Write the concepts of an OKF v0.2 bundle into a namespace via memory.set. okf-lint Lint an OKF bundle directory offline — the compile gate a mirror repository's CI runs. Exit 2 critical, 1 warnings, 0 clean. up Boot the local stack (Loom, its document extractor and ClickHouse) and print how to connect: creates .env and the ClickHouse config from their templates, runs docker compose up --build, waits for /v1/health, then prints the dashboard URL and the first memory.set (curl + Python SDK) and `claude mcp add` snippets — minting a writer token first when auth is on. Re-run any time. down Stop the local stack while preserving data; --delete-data performs a confirmed reset. ch-setup Prepare a self-managed ClickHouse or a ClickHouse Cloud service for Loom: create the loom database and the read-only reader user, or print the SQL for an admin. participant The Loom processes a provider change waits for. deployment The configuration every replica checks itself against. settings Every setting's value in force, and overrides of the live ones. secrets Credentials and signing keys stored sealed in ClickHouse. signing-key The token signing keys beside LOOM_JWT_SECRET, kept in ClickHouse. provider-lock The provider configuration lock. export-analytics Write one UTC day of usage and a namespace snapshot to S3 as Parquet, names hashed under LOOM_ANALYTICS_EXPORT_NAME_KEY. Never replaces an object, so a repeated run is safe. sql-access Make, rescope or drop a ClickHouse user that reads a tenant's memory, conversations and request ledger through the sql_v1_* views. reembed Re-embed vectors whose embedding_model != the configured embedder (run after changing LOOM_EMBEDDING_MODEL — Loom does not auto-re-embed). Covers memory, entity, and conversation vectors. Coded vector tables keep their width: change a model of another width in Settings → Models & keys, which rebuilds them. reextract Replay a namespace's raw evidence (memory.set_from_messages turns) through a running server's write path into ANOTHER namespace — the current extractor over the stored transcripts; the source is never written. deploy-delta What a colour swap changed: the deployment's ledger rows (volume, errors, p50/p95, upstream spend per operation) and its ClickHouse statement shapes (runs, bytes, seconds) for a window on each side of the swap, with flags for what moved past a coarse threshold. entity-export Export a namespace's entity graph (co-occurrence edges + triples) to GraphML or JSON-LD for an external graph or RDF tool. Read-only; every edge carries its provenance and a note that a path is declared, not proven. audit-namespace How a namespace is being used (namespace.audit): exact-key reads vs searches, hash keys, double-encoded and oversized values, hand-rolled index rows, leases stored as memories, version churn. Read-only; prints a table (code, severity, kind, count, share, first sample) or --json. Exit 1 on a critical finding. mentor The store's advice to one agent (mentor.brief): what it called, the planes its namespace offers, the hints with their evidence, and how often past hints were followed. Read-only; prints a table or --json. Exit 1 when a warning-level hint fires. serve Boot the control plane (uvicorn). migrate Apply schema migrations against the configured CH. reconcile-status Re-emit memory_embeddings rows whose status diverged from loom.memory (superseded / conflicted / duplicate marks the pool never saw), compared and re-emitted per identity row (org, namespace, agent, user_id, session_id, key). Dry run by default (counts by status, the rows it would write); --apply writes. period-digest Write period digests — one derived semantic memory over K consecutive session digests of each identity in one namespace. Additive and idempotent. Dry run by default (plans only); --apply writes. demote-activity-narration Find a namespace's extracted facts that narrate tool activity ('User read lines 1-1000 of X', 'Claude ran a command on …') with no durable outcome and demote (confidence x0.25) or label them through memory.apply. Dry run by default; keys an agent read by name in the last --days are left alone. Never deletes. recalibrate Re-probe the embedder + run per-NS calibration immediately (otherwise this runs weekly via the recalibration worker). import-clickgap Import a ClickGap (Detector→Validator→Reviewer) SQLite export into Loom (lessons + typed entities + categories). Reembed-only; preserves kind→memory_type, per-row TTL, recall_count (→ ranking via reinforcement_count), suppression polarity, and provenance. Idempotent via deterministic keys + per-run version. import-recall Import an external vector-recall agent-memory SQLite export into Loom. Introspects the DB at runtime (schema not assumed): tables are classified by column heuristics into memories (→ memory.set) or conversation transcripts (→ conversation.append, grouped by conversation id); unmatched tables/rows are skipped, never crash. Idempotent via deterministic keys. import-brain Import a markdown agent 'brain' directory into Loom: MEMORY.md (always- injected HOT layer) → profile.set sections; memory/contexts/*.md (read-on- demand WARM layer) → knowledge.add docs. Completes the migration alongside import-recall (the deep vector layer). Missing files are skipped, never crash. claim-release Release a self-signup claim: revoke its grants, clear the owner, deactivate it. The name stays reserved. mint-token Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores). revoke-token Revoke one token by jti or subject (writes loom.revoked_tokens through the revocation module; needs the configured ClickHouse). sso Carry password accounts over to single sign-on. bot Bots and CI jobs that sign in with a key or their CI's identity. doctor Diagnose pre-boot misconfig (Python version, settings, JWT, CH reachability) and print the problem list /v1/health carries, each with its fix. With --client, also examine this machine as a coding-agent client. report Print the bug form's Environment field: versions and the `loom doctor` checks. drops Print the memory ingest drop ledger (loom.memory_drops) for a namespace, by stage/reason — what the pipeline discarded, truncated or gave up on. namespaces-gc List (default) or erase throwaway namespaces — finished bench runs under the bench prefixes that nobody registered and nobody has written to for --idle-hours — in one list-scoped erasure saga per batch. erasure-re-verify Re-verify failed erasure sagas: print each one's last step-named reason and a fresh residue census (default, read-only); --resume re-runs them from their checkpoint past the recovery worker's attempts budget; --orphan-graph lists namespaces whose entity graph outlived their memories and, with --erase, purges them. reconcile After a restore of ClickHouse or Postgres, run again in the restored store the erasures and redactions the other records past the restore point, and (after a ClickHouse restore) publish again every revision committed after it. Stop every server and worker first. Dry run by default. router The router in front of a deployment's cells. cell Operate a deployment's cells through the router's directory. tracker-probe Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. tracker-contract-check Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? Names missing tables / columns, optional tables present, feature columns. Exit 0 when every checked source meets it. tracker-dupcheck-fit Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. tracker-embed-reset Clear the tracker_embed pass cursors of one tracker namespace so the next pass re-walks the window and re-embeds every item whose text recipe changed. Dry run by default (prints the cursors); --apply clears them. namespaces-adopt Give namespaces that hold live memory and no registry row an owner row: kind=bench under the bench prefixes, else memory; retention_days 0, no TTL. Dry run unless --apply. sensitivity-sweep Label live rows the sensitivity detector would have labelled on write but a path skipped. Dry run by default: counts by level, probe shape and namespace. --apply writes one new version per row, tombstones a high row's embedding rows and tags a medium row's. The closing line is the high- sensitivity invariant (high rows with a live embedding). integrity-repair Read the row-level integrity gauges (dim-0 rows holding a vector, tombstoned_at/status drift, active rows past valid_to) and, with --apply, re-emit the dim-0 rows with their true dimension so search can return them again. Drift and valid_to are listed, not repaired. rechunk Re-chunk and re-embed a namespace's knowledge documents from stored content through the current chunker, so every document carries section metadata, chunk context and document frequency. Idempotent; one embedding per new chunk. Dry run by default (counts the documents and chunks); --apply re-chunks. export-actions Export the action schema derived from the operation registry — an MCP tools/list manifest with loom extras (json) or the Cedar schema (cedar) a policy gateway governs the surface with. lexical-stats-rebuild Recompute BM25 document frequency for a namespace from its live rows (the term-frequency views only see inserts; supersession and tombstones leave stale vocabulary behind). version Print installed Loom version. install Wire Loom into a local AI client's config. Writes the MCP server block into Claude Desktop / Cursor / VS Code's config file, atomically, merging with anything already there; for Claude Code it runs (or prints) `claude mcp add` and for Codex `codex mcp add`. Restart the client afterwards. demo The first five minutes: ingest this repo's AGENTS.md (or --file) into the knowledge plane the way `make ingest` does, ask it a question, and print the answer with its evidence rows. On the stock fake-LLM stack it prints the top knowledge.search hits and says what a real answer needs. init Create an identity on a Loom server. `--agent NAME` signs an agent up by itself (server needs LOOM_SELF_SIGNUP=open): mints a writer token on a fresh agent- namespace, saves it to ~/.loom/token, and prints the claim code a person enters once, with a password of their own, to claim the namespace. ingest-transcript Pipe a chat-history file (markdown / JSON) into Loom's conversation table as one thread. Generic ingest for tools without an MCP integration (Aider, Continue.dev, JetBrains AI, anything else): export the chat → run this command → the thread shows in the dashboard's Conversations. memory-search Run a memory.search from the shell. Default human output; --json-agent emits a structured envelope agents can pipe through (a `{command, results}` JSON shape). set-host Designate the Loom host(s) for this deployment. Writes the given login handle(s) or email(s) to LOOM_PLATFORM_ADMINS in the .env (--env-file, default ./.env): a handle promotes that password account to platform_admin, an email promotes an SSO sign-in with that verified email. Multiple OK (on-call rotation, co-operators, etc.). Re-login required after to pick up the role. login Sign in to a Loom server on any browser with a one-time code. logout End the stored session on the server and forget it here. whoami Show who the stored session signs you in as. call Call any operation or API route with the stored session (`loom call --help`). token Make, list and revoke your own tokens for CI and agents (`loom token --help`).", "options: -h, --help show this help message and exit --version show program's version number and exit" ], "loom code": [ diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 8216d4de36..d689f7fdf6 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -15607,7 +15607,7 @@ }, { "dest": "mint-token", - "help": "Mint a JWT locally (requires LOOM_JWT_SECRET).", + "help": "Mint a JWT locally (requires LOOM_JWT_SECRET and verified stores).", "metavar": "mint-token" }, { diff --git a/tests/unit/test_cell_cli_orgs.py b/tests/unit/test_cell_cli_orgs.py index 6d14350e03..2e1fc087ee 100644 --- a/tests/unit/test_cell_cli_orgs.py +++ b/tests/unit/test_cell_cli_orgs.py @@ -475,6 +475,27 @@ async def connect(self) -> None: assert not [r for r in caplog.records if r.name == "clickhouse_connect"] +@pytest.mark.parametrize("diagnostic_mode", ["open", "recording", "enforced", "partial"]) +def test_setup_identity_probe_neither_installs_nor_uses_acceptance( + monkeypatch: pytest.MonkeyPatch, diagnostic_mode: str +) -> None: + """Bootstrap checks store ownership, never a reusable data-plane admission.""" + from server.core import retirement + + _cell_set(monkeypatch, (diagnostic_mode, "eu-1")) + client = AsyncMock() + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(pg_profile, "recorded_authority", AsyncMock(return_value=None)) + acceptance = AsyncMock(side_effect=AssertionError("bootstrap is not admission")) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + monkeypatch.setattr(cell, "refresh", acceptance) + before = cell._view + asyncio.run(_ch.check_setup_stores()) + assert cell._view is before + assert not retirement.get_gate().enrolled + acceptance.assert_not_awaited() + + def test_the_loom_code_drain_after_a_subcommand_is_a_read() -> None: """The reconnect that drains buffered writes runs after every `loom code` subcommand, reads included; the subcommand's own connect gated its writes, so the diff --git a/tests/unit/test_cli_hygiene.py b/tests/unit/test_cli_hygiene.py index 5458194131..630ca7d2de 100644 --- a/tests/unit/test_cli_hygiene.py +++ b/tests/unit/test_cli_hygiene.py @@ -171,7 +171,7 @@ def test_mint_token_signs_with_the_keyring_it_reads( pyjwt.decode(token, "k" * 40, algorithms=["HS256"], audience=settings.jwt_audience) -def test_an_offline_mint_signs_with_the_root_secret( +def test_an_offline_native_mint_refuses_without_enrollment( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _auth_on_with(monkeypatch, "a-real-secret-that-is-not-the-default") @@ -180,10 +180,9 @@ def _unreachable(run: Any, **_declared: object) -> None: raise ConnectionError("no clickhouse") monkeypatch.setattr(access, "run_connected", _unreachable) - assert cli.main(_MINT) == 0 + assert cli.main(_MINT) == 1 out, err = capsys.readouterr() - assert "signing keyring not read; signing as kid" in err - assert pyjwt.get_unverified_header(out.strip())["kid"] == "default" + assert out == "" and "enrolled scope" in err def _doctor(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> tuple[int, str]: @@ -324,8 +323,13 @@ def _refuse(_run: object, **_declared: object) -> bool: raise AuthorityMismatch("this deployment's memory authority is postgres") monkeypatch.setattr(access, "run_connected", _refuse) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers()) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--platform-admin", "--ttl-days", "1"] + ) with pytest.raises(AuthorityMismatch): - access._platform_admin_rollout_ready() + access._cmd_mint_token(args) def test_a_revocation_for_another_deployment_is_refused_as_such( diff --git a/tests/unit/test_native_command_retirement.py b/tests/unit/test_native_command_retirement.py new file mode 100644 index 0000000000..228352a83d --- /dev/null +++ b/tests/unit/test_native_command_retirement.py @@ -0,0 +1,201 @@ +"""The actual native command cannot outlive its certified participant.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch, access +from server.core import retirement +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server.core import background, ch, pg, pg_profile + from server.core.model_work_gate import ModelWorkGate + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + + async def connect(): + client.connected = True + + async def disconnect(): + client.connected = False + + client.connect, client.disconnect = connect, disconnect + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", AsyncMock()) + + async def verified(_): + assert not gate.enrolled + events.append("verified") + + async def enroll(): + assert events == ["verified"] + gate.enrolled = True + events.append("enrolled") + + async def acceptance(**_): + assert gate.enrolled and gate.active > 0 and not gate.closed + events.append("acceptance") + + async def certificate(proof): + assert gate.quiescent and client.connected + assert events[-1] == "buffer" + events.append("certificate") + + async def flush(): + events.append("buffer") + return True + + monkeypatch.setattr(pg_profile, "check_configured", verified) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(_ch, "check_acceptance", acceptance) + model_gate = ModelWorkGate() + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(command, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(barrier, "drain_task_sets", background.drain_task_sets) + monkeypatch.setattr(command, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "flush_registered_buffers", flush) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "record_retirement", certificate) + return gate, events + + +def test_actual_claim_release_keeps_lookup_and_release_in_one_lifetime(native, monkeypatch): + import asyncio + + from server.core.background import create_deferred_task + from server.identity.app import claim_release + from server.identity.store import namespaces + + gate, events = native + + async def lookup(*_, **__): + assert gate.active > 0 and not gate.closed + events.append("lookup") + return {"owner_email": "owner@example.test"} + + async def release(**_): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append("release") + + async def descendant(): + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("descendant") + + async def write(): + await asyncio.sleep(0) + assert gate.enrolled and gate.closed + assert "certificate" not in events + events.append("deferred") + create_deferred_task(descendant()) + + create_deferred_task(write()) + return SimpleNamespace(grants_revoked=2) + + monkeypatch.setattr(namespaces, "fetch_row", lookup) + monkeypatch.setattr(claim_release, "release_claim", release) + assert access._cmd_claim_release(SimpleNamespace(org="o", namespace="n", yes=True)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("lookup") < events.index("release") < events.index("certificate") + assert events.index("release") < events.index("deferred") < events.index("descendant") + assert events.index("descendant") < events.index("buffer") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("platform_admin", [False, True]) +def test_actual_mint_readiness_keyring_signing_and_audit_precede_retirement( + native, monkeypatch, platform_admin +): + import argparse + + from server.auth import jwt, keyring, machines + from server.observability import audit + + gate, events = native + + def observed(event): + assert gate.active > 0 and gate.enrolled and not gate.closed + assert "certificate" not in events + events.append(event) + + async def ready(_): + observed("readiness") + return True + + async def refresh(**_): + observed("keyring") + return True + + def mint(**_): + observed("mint") + return "test-token" + + async def record(*_, **__): + observed("audit") + return "audit-id" + + monkeypatch.setattr(machines, "platform_admin_tokens_ready", ready) + monkeypatch.setattr(keyring, "refresh", refresh) + monkeypatch.setattr(jwt, "mint_token", mint) + monkeypatch.setattr(audit, "emit", record) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + argv = ["mint-token", "--subject", "sa:test", "--org", "o", "--ttl-days", "1"] + argv += ["--platform-admin"] if platform_admin else ["--namespaces", "n"] + assert access._cmd_mint_token(parser.parse_args(argv)) == 0 + assert events.count("enrolled") == events.count("certificate") == 1 + assert events.index("keyring") < events.index("mint") < events.index("certificate") + if platform_admin: + assert events.index("readiness") < events.index("keyring") + assert events.index("mint") < events.index("audit") < events.index("certificate") + assert gate.quiescent + + +@pytest.mark.parametrize("failure", ["unavailable", "wrong-pair"]) +def test_actual_mint_never_signs_without_verified_enrollment(native, monkeypatch, failure): + import argparse + from unittest.mock import Mock + + from server.auth import jwt + from server.core import pg_profile + from server.core.pg import AuthorityMismatch + + gate, events = native + + async def refuse(_): + if failure == "wrong-pair": + raise AuthorityMismatch("different paired stores") + raise ConnectionError("unavailable") + + minted = Mock(side_effect=AssertionError("unenrolled mint")) + monkeypatch.setattr(pg_profile, "check_configured", refuse) + monkeypatch.setattr(jwt, "mint_token", minted) + parser = argparse.ArgumentParser() + access._add_mint_token_parser(parser.add_subparsers(dest="cmd")) + args = parser.parse_args( + ["mint-token", "--subject", "sa:test", "--org", "o", "--namespaces", "n"] + ) + if failure == "wrong-pair": + with pytest.raises(AuthorityMismatch): + access._cmd_mint_token(args) + else: + assert access._cmd_mint_token(args) == 1 + minted.assert_not_called() + assert not gate.enrolled and not gate.closed and events == [] diff --git a/tests/unit/test_platform_admin_bots.py b/tests/unit/test_platform_admin_bots.py index 84fd4c1834..ed2ddb14b2 100644 --- a/tests/unit/test_platform_admin_bots.py +++ b/tests/unit/test_platform_admin_bots.py @@ -13,6 +13,7 @@ import asyncio import json from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -181,13 +182,16 @@ def minting(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: monkeypatch.setenv("LOOM_JWT_SECRET", _SECRET) monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + from server.auth import keyring + + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) # These mint tests run after every replica has the human-admin guards. - monkeypatch.setattr(access, "_platform_admin_rollout_ready", lambda: True) + monkeypatch.setattr(access, "_platform_admin_rollout_ready", AsyncMock(return_value=True)) recorded: list[dict[str, Any]] = [] - def audited(args: Any, summary: dict[str, Any]) -> bool: + async def audited(args: Any, summary: dict[str, Any]) -> bool: recorded.append({"subject": args.subject, "jti": summary["jti"]}) return True @@ -236,7 +240,7 @@ def test_a_platform_admin_mint_the_audit_log_misses_is_not_handed_out( ) -> None: from server.cli import access - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda args, summary: False) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=False)) assert cli.main([*_MINT, "--subject", "sa:cp", "--ttl-days", "7"]) == 1 out, err = capsys.readouterr() assert out == "" and "no token" in err @@ -257,7 +261,7 @@ async def emit(event: str, **kw: Any) -> str: monkeypatch.setattr(audit, "emit", emit) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) args = SimpleNamespace(subject="sa:cp", org="acme", ttl_days=7) - assert access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]}) + assert asyncio.run(access._audit_platform_admin_mint(args, {"jti": "j1", "namespaces": ["*"]})) [(event, kw)] = rows assert event == "auth.platform_admin_minted" and kw["org"] == "acme" assert kw["subject"] == {"subject": "sa:cp", "jti": "j1", "namespaces": ["*"], "ttl_days": 7} @@ -266,7 +270,9 @@ async def lost(event: str, **kw: Any) -> str: return "" # audit.emit's answer when its insert failed monkeypatch.setattr(audit, "emit", lost) - assert not access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + assert not asyncio.run( + access._audit_platform_admin_mint(args, {"jti": "j2", "namespaces": ["*"]}) + ) _CP_BODY = { diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 134f54cc8a..93d3a5eae5 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -4,6 +4,7 @@ import time from typing import Any +from unittest.mock import AsyncMock import pytest @@ -87,9 +88,10 @@ def mint(**kw: Any) -> str: monkeypatch.setattr(instances, "live_rows", lambda: [{"schema_version": 76}]) monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(keyring, "load_for_offline_mint", lambda **kw: None) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) - monkeypatch.setattr(access, "_audit_platform_admin_mint", lambda *a: True) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) monkeypatch.setattr(jwt, "mint_token", mint) code = cli.main( [ @@ -139,7 +141,7 @@ def mint(**kw): ] ) out, err = capsys.readouterr() - assert code == 1 and out == "" and "rollout" in err + assert code == 1 and out == "" and "enrolled scope" in err def test_admin_token_floor_matches_the_physical_migration() -> None: diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index d1ee7960e3..0e70602dd7 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -66,7 +66,7 @@ def _eperm(pid: int, sig: int) -> None: assert not pp.proves_stopped(_with(pid=_dead_pid())) -def test_a_container_restarted_into_a_new_pid_namespace_is_proven_stopped( +def test_a_changed_container_pid_namespace_does_not_prove_the_old_process_stopped( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(pp, "_container", lambda: "c" * 64) diff --git a/tests/unit/test_revoke_issued_before.py b/tests/unit/test_revoke_issued_before.py index e7e9bb0af0..fa0c71444d 100644 --- a/tests/unit/test_revoke_issued_before.py +++ b/tests/unit/test_revoke_issued_before.py @@ -20,6 +20,7 @@ from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Any +from unittest.mock import AsyncMock import jwt as pyjwt import pytest @@ -265,10 +266,12 @@ def test_cli_mint_reports_the_jti_but_never_beside_a_saved_token( from server.settings import settings monkeypatch.setattr(settings, "jwt_secret", _SECRET, raising=False) + from server.auth import keyring from server.cli import access - # An empty keyring: the mint signs with the root secret, reading no ClickHouse. - monkeypatch.setattr(access, "run_connected", lambda run, **_declared: None) + monkeypatch.setattr(access, "run_connected", lambda run, **_declared: asyncio.run(run(None))) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) rc = cli.main( ["mint-token", "--subject", "sa:x", "--org", "acme", "--namespaces", "shop", "--json"] ) @@ -539,6 +542,6 @@ def test_cli_mint_refuses_a_reserved_subject_without_a_traceback( def _read(**_k: Any) -> None: raise AssertionError("the keyring was read for a subject no token may carry") - monkeypatch.setattr(keyring, "load_for_offline_mint", _read) + monkeypatch.setattr(keyring, "load_for_connected_mint", _read) rc = cli.main(["mint-token", "--subject", subject, "--org", "acme", "--namespaces", "shop"]) assert rc == 2 and "grant principal" in capsys.readouterr().err From 6badd7de542684cf508bd0745c947fc8e43d2b49 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 00:38:19 +0000 Subject: [PATCH 094/181] Check the participant identity in cell heartbeat tests Keep the full heartbeat assertion aligned with the provider identity used to join serving replacements to their retirement records. Signed-off-by: zlareb1 --- server/core/instances.py | 3 ++- tests/unit/test_cell.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/core/instances.py b/server/core/instances.py index ecbe666c07..42109b0d35 100644 --- a/server/core/instances.py +++ b/server/core/instances.py @@ -139,7 +139,8 @@ def cell_row() -> dict[str, Any]: """What this process reports about its cell (`loom.cell_heartbeats`). Beside the fingerprint's version and fields, its JSON carries keys no fingerprint compares (docs/architecture/cells.md, "The heartbeat's keys"): `serves` (LOOM_SERVES_TRAFFIC), - `marked` (whether this process's pooled sessions keep the tenancy marker, null until a + `participant_id` (this process's registered provider identity), `marked` + (whether this process's pooled sessions keep the tenancy marker, null until a cell-state read lands), and with LOOM_CELL set, until the `first` row is read and then on a first cell only, `router` and `router_why` (`cell_router.router_key`).""" from server.core.provider_process import process_identity diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index dcd2536ac4..624798a45a 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -7,13 +7,14 @@ import json import time from unittest.mock import AsyncMock, call +from uuid import UUID import pytest from pydantic import ValidationError from server import boot, worker_main from server.auth import jwt -from server.core import cell, cell_probe, instances, pg, pg_profile +from server.core import cell, cell_probe, instances, pg, pg_profile, provider_process from server.core import ch as ch_mod from server.settings import Settings, settings from server.transport.model_maintenance import ModelMaintenanceMiddleware @@ -329,6 +330,8 @@ def test_an_on_process_outside_a_cell_holds_from_its_first_step( def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( monkeypatch: pytest.MonkeyPatch, ) -> None: + participant = UUID(int=7) + monkeypatch.setattr(provider_process, "process_identity", lambda: participant) monkeypatch.setattr(instances.readiness, "draining", lambda: False) monkeypatch.setattr(instances.deployment_config, "drift", lambda: []) monkeypatch.setattr(settings, "serves_traffic", True) @@ -346,6 +349,7 @@ def test_the_heartbeat_reports_the_cell_and_a_gated_process_is_not_serving( **cell.as_value(cell.mine()), "serves": True, "marked": None, + "participant_id": str(participant), } From ce81164cce96865f546c7c238817dc1ff524ed6e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:05:44 +0000 Subject: [PATCH 095/181] Native nonce cleanup keeps the first failure Prepare and attach attempt nonce deletion without replacing the failure that selected cleanup. Standalone cleanup interruptions still propagate. Remove the unused removal snapshot argument and unreachable success-test branches. Signed-off-by: zlareb1 --- server/cli/cell.py | 3 +- server/cli/cell_attach.py | 15 +- server/cli/cell_prepare.py | 6 +- .../unit/test_cell_native_http_first_error.py | 6 +- .../test_cell_native_nonce_first_error.py | 269 ++++++++++++++++++ .../unit/test_cell_place_http_first_error.py | 7 +- 6 files changed, 289 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_cell_native_nonce_first_error.py diff --git a/server/cli/cell.py b/server/cli/cell.py index a6fb4e21e5..d583fb48dc 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -349,7 +349,7 @@ def _cmd_remove(args: argparse.Namespace) -> int: print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 try: - answer = asyncio.run(_remove_locked(args, router, row)) + answer = asyncio.run(_remove_locked(args, router)) except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) return 1 @@ -360,7 +360,6 @@ def _cmd_remove(args: argparse.Namespace) -> int: async def _remove_locked( args: argparse.Namespace, router: httpx.URL, - row: dict[str, Any], transport: httpx.AsyncBaseTransport | None = None, ) -> dict[str, Any]: from server.cli import cell_control diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8fd6e7ebee..821a77448b 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -376,7 +376,11 @@ async def begin(self) -> None: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) # -- the scans --------------------------------------------------------------------- @@ -542,11 +546,16 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.mutate( + answer = await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) + return answer # -- the run ----------------------------------------------------------------------- diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4310b4c9bb..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -513,7 +513,11 @@ async def _prepare_locked( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py index 4634e7d69c..c9b61bd146 100644 --- a/tests/unit/test_cell_native_http_first_error.py +++ b/tests/unit/test_cell_native_http_first_error.py @@ -180,11 +180,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) + expected = "dep-eu-1" if boundary == "health" else exchange.answer assert result == expected assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_cell_place_http_first_error.py b/tests/unit/test_cell_place_http_first_error.py index 3587eb95ba..b8120702c9 100644 --- a/tests/unit/test_cell_place_http_first_error.py +++ b/tests/unit/test_cell_place_http_first_error.py @@ -62,12 +62,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) - assert result == expected + assert result is None assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] From 06355a3b74eb2bb5ae95505b3d98a07cc8811183 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:35:26 +0000 Subject: [PATCH 096/181] Bind trigger timing receipts to distinct API and CLI configurations The serving API keeps its cell token while the native writing CLI requires it empty. Bind both complete configurations and their common projection excluding only that token, with closed schema 2 receipts and fixed credential posture. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 150 ++++- tests/unit/test_bench_cell_trigger_gate.py | 567 +++++++++++++++++- .../test_cell_trigger_gate_artifact_close.py | 19 +- .../test_cell_trigger_gate_diagnostics.py | 15 +- .../test_cell_trigger_gate_first_error.py | 13 +- 5 files changed, 717 insertions(+), 47 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 32803bd38d..19db375e4b 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -123,6 +123,7 @@ def _digest(value: Any) -> str: class Fixture: """Nonsecret receipt from the owner that created and launched this exact fixture.""" + schema: int base_url: str pg_host: str pg_port: int @@ -141,22 +142,36 @@ class Fixture: source_tree: str launch_manifest_sha256: str ownership_receipt_sha256: str - configuration_sha256: str + api_configuration_sha256: str + cli_configuration_sha256: str + shared_configuration_sha256: str background_workers: str controller_excluded: bool - pg_internal_port: int | None = None + pg_internal_port: int | None @classmethod def read(cls, path: str) -> Fixture: try: - with Path(path).open("rb") as manifest_file: - raw = manifest_file.read(16_385) - _require(len(raw) <= 16_384, "manifest size") - return cls(**json.loads(raw)) + value, _ = _receipt(path, max_bytes=16_384) + _require(set(value) == set(cls.__dataclass_fields__), "fixture fields") + _require(type(value["schema"]) is int and value["schema"] == 2, "fixture schema") + return cls(**value) except (TypeError, ValueError, OSError): raise Invalid("fixture manifest") from None def validate(self, args: argparse.Namespace, config: Any) -> int: + _require(type(self.schema) is int and self.schema == 2, "fixture schema") + integers = {"schema", "pg_port", "ch_port", "epoch"} + _require( + all(type(getattr(self, key)) is int for key in integers) + and type(self.controller_excluded) is bool + and all( + type(value) is str + for key, value in asdict(self).items() + if key not in integers | {"controller_excluded", "pg_internal_port"} + ), + "fixture field types", + ) supplied = _endpoint(args.pg_dsn, pg=True) configured = _endpoint(config.pg_dsn, pg=True) _require(supplied == configured, "configured PG endpoint") @@ -204,11 +219,24 @@ def validate(self, args: argparse.Namespace, config: Any) -> int: for value in ( self.launch_manifest_sha256, self.ownership_receipt_sha256, - self.configuration_sha256, + self.api_configuration_sha256, + self.cli_configuration_sha256, + self.shared_configuration_sha256, ): - _require(bool(_SHA.fullmatch(value)), "fixture receipt digest") + _require(type(value) is str and bool(_SHA.fullmatch(value)), "fixture receipt digest") + actual = _configuration(config) + _require( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "empty native CLI credentials", + ) + _require( + self.api_configuration_sha256 != self.cli_configuration_sha256, + "distinct API and CLI configurations", + ) + _require(self.cli_configuration_sha256 == _digest(actual), "fixture CLI configuration") _require( - self.configuration_sha256 == _digest(_configuration(config)), "fixture configuration" + self.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "fixture shared configuration", ) _require(type(self.epoch) is int and self.epoch > 0, "authority epoch") _require( @@ -221,8 +249,17 @@ def validate(self, args: argparse.Namespace, config: Any) -> int: def _configuration(config: Any) -> dict[str, Any]: - # A complete digest catches a setting change without publishing credentials. - return cast(dict[str, Any], config.model_dump(mode="json")) + from server.settings import Settings + + values = config.model_dump(mode="json") + _require(type(values) is dict and set(values) == set(Settings.model_fields), "Settings fields") + return cast(dict[str, Any], values) + + +def _shared_configuration(config: Any) -> dict[str, Any]: + values = dict(_configuration(config)) + del values["cell_router_token"] + return values def _source(fixture: Fixture) -> dict[str, Any]: @@ -273,7 +310,15 @@ def _workload(config: Any) -> dict[str, Any]: } -def _receipt(path: str) -> tuple[dict[str, Any], str]: +def _receipt_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + _require(key not in result, "duplicate receipt field") + result[key] = value + return result + + +def _receipt(path: str, *, max_bytes: int = 65_536) -> tuple[dict[str, Any], str]: fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) with os.fdopen(fd, "rb") as source: metadata = os.fstat(source.fileno()) @@ -281,9 +326,9 @@ def _receipt(path: str) -> tuple[dict[str, Any], str]: stat.S_ISREG(metadata.st_mode) and metadata.st_uid == os.geteuid(), "owned regular receipt", ) - raw = source.read(65_537) - _require(len(raw) <= 65_536, "receipt size") - value = json.loads(raw) + raw = source.read(max_bytes + 1) + _require(len(raw) <= max_bytes, "receipt size") + value = json.loads(raw, object_pairs_hook=_receipt_object) _require(isinstance(value, dict), "receipt object") return value, hashlib.sha256(raw).hexdigest() @@ -325,6 +370,52 @@ def _owned_server(server: dict[str, Any]) -> None: _require(_start_ticks(pid) == birth, "stable process birth") +_CREDENTIAL_POSTURE = { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, +} + + +def _attested_configuration(configuration: dict[str, Any], fixture: Fixture, config: Any) -> None: + from server.core import deployment_config + + hashes = { + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + } + _require( + set(configuration) == {"schema", "workload", "fingerprints", "credential_posture", *hashes} + and type(configuration["schema"]) is int + and configuration["schema"] == 2, + "configuration artifact schema", + ) + _require(all(configuration[key] == value for key, value in hashes.items()), "launch digests") + _require( + fixture.cli_configuration_sha256 == _digest(_configuration(config)), + "launch CLI configuration", + ) + _require( + fixture.shared_configuration_sha256 == _digest(_shared_configuration(config)), + "launch shared configuration", + ) + _require( + type(configuration["credential_posture"]) is dict + and all(type(value) is bool for value in configuration["credential_posture"].values()) + and configuration["credential_posture"] == _CREDENTIAL_POSTURE, + "launch credential posture", + ) + _require( + _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" + ) + _require( + configuration["fingerprints"] == deployment_config.fingerprints(config), + "launch configuration fingerprints", + ) + + def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str, Any]: from server.core import deployment_config @@ -333,7 +424,7 @@ def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str _require( set(receipt) == {"schema", "server", "source", "configuration_artifact"} and type(receipt["schema"]) is int - and receipt["schema"] == 1, + and receipt["schema"] == 2, "launch receipt schema", ) server, source, artifact = ( @@ -356,23 +447,8 @@ def _launch(args: argparse.Namespace, fixture: Fixture, config: Any) -> dict[str ) configuration, artifact_digest = _receipt(artifact["path"]) _require(artifact_digest == artifact["sha256"], "configuration artifact hash") - _require( - set(configuration) == {"schema", "configuration_sha256", "workload", "fingerprints"} - and type(configuration["schema"]) is int - and configuration["schema"] == 1, - "configuration artifact schema", - ) - _require( - configuration["configuration_sha256"] - == fixture.configuration_sha256 - == _digest(_configuration(config)), - "launch effective configuration", - ) - _require( - _digest(configuration["workload"]) == _digest(_workload(config)), "launch workload settings" - ) + _attested_configuration(configuration, fixture, config) fingerprints = deployment_config.fingerprints(config) - _require(configuration["fingerprints"] == fingerprints, "launch configuration fingerprints") _owned_server(server) return { "health_instance": server["health_instance"], @@ -604,7 +680,12 @@ async def _proof( "epoch": fixture.epoch, "instance_id": fixture.instance_id, "fingerprint_sha256": _digest(current), - "configuration_sha256": fixture.configuration_sha256, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "shared_configuration_excludes": ["cell_router_token"], + "credential_posture": _CREDENTIAL_POSTURE, + "api_configuration_attested": True, "source": source, "launch": launch, } @@ -1469,6 +1550,9 @@ def main() -> int: artifact.emit( "configuration", manifest=asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=_CREDENTIAL_POSTURE, + api_configuration_attested=True, source=source, launch=launch, manifest_sha256=_digest(asdict(fixture)), diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 211dfd88c9..9d9ce8f180 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -42,8 +42,13 @@ def args(**overrides): ) -def fixture_config(): - config = SimpleNamespace( +def role_config(): + from server.settings import Settings + + config = SimpleNamespace(**Settings.model_construct().model_dump(mode="json")) + vars(config).update( + cell_router_token="", + router_operator_token="", pg_dsn=args().pg_dsn, ch_host="127.0.0.1", ch_port=18790, @@ -74,7 +79,13 @@ def dump(**kwargs): } config.model_dump = dump + return config + + +def fixture_config(): + config = role_config() manifest = gate.Fixture( + schema=2, base_url=args().base_url, pg_host="127.0.0.1", pg_port=15790, @@ -93,7 +104,13 @@ def dump(**kwargs): source_tree="b" * 40, launch_manifest_sha256="c" * 64, ownership_receipt_sha256="d" * 64, - configuration_sha256=gate._digest(config.model_dump()), + api_configuration_sha256=gate._digest( + {**config.model_dump(), "cell_router_token": "lcr_" + "0" * 128} + ), + cli_configuration_sha256=gate._digest(config.model_dump()), + shared_configuration_sha256=gate._digest( + {k: v for k, v in config.model_dump().items() if k != "cell_router_token"} + ), background_workers="off", controller_excluded=True, pg_internal_port=5432, @@ -1247,7 +1264,21 @@ async def send(self, request, **kwargs): await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) else: proof = await gate._proof(PG(), CH(), args(launch_manifest=str(launch_path)), fixture) - assert proof["configuration_sha256"] == fixture.configuration_sha256 + assert proof["api_configuration_sha256"] == fixture.api_configuration_sha256 + assert proof["cli_configuration_sha256"] == fixture.cli_configuration_sha256 + assert proof["shared_configuration_sha256"] == fixture.shared_configuration_sha256 + assert proof["shared_configuration_excludes"] == ["cell_router_token"] + assert proof["credential_posture"] == receipt_posture() + assert proof["api_configuration_attested"] is True + + +def receipt_posture(): + return { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + } def launch_files(tmp_path, fixture, config): @@ -1280,8 +1311,11 @@ def launch_files(tmp_path, fixture, config): } } configuration = { - "schema": 1, - "configuration_sha256": fixture.configuration_sha256, + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "credential_posture": receipt_posture(), "workload": public, "fingerprints": deployment_config.fingerprints(config), } @@ -1290,7 +1324,7 @@ def launch_files(tmp_path, fixture, config): artifact_path.write_bytes(artifact_bytes) source_path = str(__import__("pathlib").Path(gate.__file__).resolve().parents[1]) launch = { - "schema": 1, + "schema": 2, "server": { "pid": 4242, "start_ticks": "123", @@ -1339,7 +1373,7 @@ def test_launch_receipt_binds_named_owned_process_settings_and_nonsecret_artifac monkeypatch.setattr(gate, "_owned_server", lambda server: checked.append(server), raising=False) if fault in {"config", "workload", "workload_type", "fingerprints"}: if fault == "config": - configuration["configuration_sha256"] = "0" * 64 + configuration["cli_configuration_sha256"] = "0" * 64 if fault == "workload": configuration["workload"]["pg_delivery_batch_rows"] = 501 if fault == "workload_type": @@ -1690,3 +1724,520 @@ async def test_successful_restoration_still_fails_when_artifact_write_fails(): await gate._cleanup(triggers, conn, artifact) assert observed.value is first conn.close.assert_awaited_once() + + +def rewrite_launch(fixture, path, artifact_path, launch, configuration, *, raw=None, level=None): + configuration_raw = raw if level == "configuration" else json.dumps(configuration).encode() + artifact_path.write_bytes(configuration_raw) + launch["configuration_artifact"]["sha256"] = gate.hashlib.sha256(configuration_raw).hexdigest() + launch_raw = raw if level == "launch" else json.dumps(launch).encode() + path.write_bytes(launch_raw) + return dataclasses.replace( + fixture, launch_manifest_sha256=gate.hashlib.sha256(launch_raw).hexdigest() + ) + + +def test_schema_two_local_pair_has_only_one_fixed_credential_difference(monkeypatch, tmp_path): + from server.core import deployment_config + + fixture, config = fixture_config() + fixture, path, _, _, configuration = launch_files(tmp_path, fixture, config) + checked = [] + monkeypatch.setattr(gate, "_owned_server", lambda value: checked.append(value)) + assert fixture.validate(args(), config) == 45500 + assert gate._launch(args(launch_manifest=str(path)), fixture, config) + actual = config.model_dump(mode="json") + api = {**actual, "cell_router_token": "lcr_" + "0" * 128} + assert gate._digest(api) == fixture.api_configuration_sha256 + assert gate._digest(actual) == fixture.cli_configuration_sha256 + assert fixture.api_configuration_sha256 != fixture.cli_configuration_sha256 + assert gate._shared_configuration(config) == { + key: value for key, value in api.items() if key != "cell_router_token" + } + assert configuration["fingerprints"] == deployment_config.fingerprints(config) + assert len(checked) == 1 + + +@pytest.mark.parametrize("field", ["cell_router_token", "router_operator_token", "jwt_secret"]) +@pytest.mark.parametrize("shape", ["missing", "extra"]) +def test_complete_settings_shape_refuses_missing_or_extra_fields(field, shape): + config = role_config() + if shape == "missing": + delattr(config, field) + else: + config.unexpected_setting = "not-an-exclusion" + with pytest.raises(gate.Invalid, match="Settings fields"): + gate._configuration(config) + + +def test_every_other_effective_setting_stays_in_the_common_digest(): + from server.settings import Settings + + fixture, config = fixture_config() + original = config.model_dump(mode="json") + fields = set(original) - {"cell_router_token"} + assert fields == set(Settings.model_fields) - {"cell_router_token"} + configuration = { + "schema": 2, + "api_configuration_sha256": fixture.api_configuration_sha256, + "cli_configuration_sha256": fixture.cli_configuration_sha256, + "shared_configuration_sha256": fixture.shared_configuration_sha256, + "workload": {}, + "fingerprints": {}, + "credential_posture": receipt_posture(), + } + for field in sorted(fields): + try: + setattr(config, field, ["changed-type-and-value", original[field]]) + changed = dataclasses.replace( + fixture, cli_configuration_sha256=gate._digest(config.model_dump(mode="json")) + ) + configuration["cli_configuration_sha256"] = changed.cli_configuration_sha256 + with pytest.raises(gate.Invalid, match="launch shared configuration"): + gate._attested_configuration(configuration, changed, config) + finally: + setattr(config, field, original[field]) + + +@pytest.mark.parametrize("value", [True, 1, 1.0, "1", None, [], {}]) +def test_common_digest_keeps_json_value_types(value): + _, config = fixture_config() + config.pg_delivery_batch_rows = value + assert gate._digest(gate._shared_configuration(config)) != gate._digest( + {**gate._shared_configuration(config), "pg_delivery_batch_rows": "unchanged"} + ) + hashes = {gate._digest({"value": v}) for v in [True, 1, 1.0, "1", None, [], {}]} + assert len(hashes) == 7 + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("version", [None, 1, True, 2.0, "2"]) +def test_old_missing_and_noninteger_receipt_versions_refuse( + monkeypatch, tmp_path, location, version +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + target = {"fixture": manifest, "launch": launch, "configuration": configuration}[location] + if version is None: + del target["schema"] + else: + target["schema"] = version + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_text(json.dumps(manifest)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize( + "location", + [ + "fixture", + "launch", + "server", + "source", + "configuration_artifact", + "configuration", + "workload", + "fingerprints", + "credential_posture", + ], +) +@pytest.mark.parametrize("fault", ["duplicate", "unknown", "missing"]) +def test_closed_receipt_objects_reject_duplicate_and_unknown_fields( + monkeypatch, tmp_path, location, fault +): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + root = ( + manifest + if location == "fixture" + else configuration + if location in {"configuration", "workload", "fingerprints", "credential_posture"} + else launch + ) + target = root if location in {"fixture", "launch", "configuration"} else root[location] + if fault == "unknown": + target["unexpected"] = True + blob = json.dumps(root).encode() + elif fault == "missing": + del target[next(iter(target))] + blob = json.dumps(root).encode() + else: + key = next(iter(target)) + original = json.dumps(target) + duplicate = original[:-1] + ", " + json.dumps(key) + ": " + json.dumps(target[key]) + "}" + blob = json.dumps(root).replace(original, duplicate, 1).encode() + if location != "server": + monkeypatch.setattr( + gate, "_owned_server", lambda _: pytest.fail("invalid receipt reached birth") + ) + if location == "fixture": + path.write_bytes(blob) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch( + fixture, + path, + artifact_path, + launch, + configuration, + raw=blob, + level="configuration" if root is configuration else "launch", + ) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("key", list(receipt_posture())) +@pytest.mark.parametrize("fault", ["wrong", "integer", "missing"]) +def test_posture_requires_exact_boolean_literals(monkeypatch, tmp_path, key, fault): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + if fault == "missing": + del configuration["credential_posture"][key] + else: + configuration["credential_posture"][key] = ( + not receipt_posture()[key] if fault == "wrong" else int(receipt_posture()[key]) + ) + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + monkeypatch.setattr(gate, "_owned_server", lambda _: pytest.fail("bad posture reached birth")) + with pytest.raises(gate.Invalid, match="credential posture"): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("fault", ["equal", "cli", "shared", "swapped"]) +def test_role_digests_cannot_be_equal_swapped_or_mismatched(fault): + fixture, config = fixture_config() + if fault == "equal": + fixture = dataclasses.replace( + fixture, api_configuration_sha256=fixture.cli_configuration_sha256 + ) + elif fault == "swapped": + fixture = dataclasses.replace( + fixture, + api_configuration_sha256=fixture.cli_configuration_sha256, + cli_configuration_sha256=fixture.api_configuration_sha256, + ) + else: + fixture = dataclasses.replace(fixture, **{fault + "_configuration_sha256": "0" * 64}) + with pytest.raises(gate.Invalid): + fixture.validate(args(), config) + + +@pytest.mark.parametrize("location", ["fixture", "configuration"]) +def test_removed_configuration_digest_is_not_a_compatibility_field(monkeypatch, tmp_path, location): + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + target = dataclasses.asdict(fixture) if location == "fixture" else configuration + target["configuration_sha256"] = fixture.cli_configuration_sha256 + if location == "fixture": + path.write_text(json.dumps(target)) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + else: + fixture = rewrite_launch(fixture, path, artifact_path, launch, configuration) + with pytest.raises(gate.Invalid): + gate._launch(args(launch_manifest=str(path)), fixture, config) + + +@pytest.mark.parametrize("location", ["fixture", "launch", "configuration"]) +def test_duplicate_json_fields_refuse_without_any_schema_assumption(tmp_path, location): + path = tmp_path / "receipt.json" + path.write_text('{"' + location + '":0,"' + location + '":1}') + with pytest.raises(gate.Invalid, match="duplicate receipt field"): + gate._receipt(str(path)) + + +@pytest.mark.parametrize( + "field", ["api_configuration_sha256", "cli_configuration_sha256", "shared_configuration_sha256"] +) +@pytest.mark.parametrize("value", [None, True, 64, "A" * 64, "f" * 63, "g" * 64]) +def test_role_digest_fields_are_lowercase_sha256_strings(field, value): + fixture, config = fixture_config() + reason = "fixture receipt digest" if type(value) is str else "fixture field types" + with pytest.raises(gate.Invalid, match=reason): + dataclasses.replace(fixture, **{field: value}).validate(args(), config) + + +def test_schema_two_requires_explicit_internal_port_and_preserves_existing_bytes(tmp_path): + fixture, config = fixture_config() + value = dataclasses.asdict(fixture) + del value["pg_internal_port"] + original = json.dumps(value).encode() + path = tmp_path / "fixture.json" + path.write_bytes(original) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == original + value["pg_internal_port"] = None + path.write_text(json.dumps(value)) + assert gate.Fixture.read(str(path)).validate(args(), config) == 45500 + + +@pytest.mark.parametrize("target", ["fixture", "launch", "configuration"]) +@pytest.mark.parametrize("fault", ["symlink", "oversize", "owner"]) +def test_all_receipt_roles_keep_file_safety(monkeypatch, tmp_path, target, fault): + import os + + fixture, config = fixture_config() + fixture, path, artifact_path, launch, configuration = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) + selected = {"fixture": manifest_path, "launch": path, "configuration": artifact_path}[target] + original = selected.read_bytes() + if fault == "symlink": + saved = tmp_path / "saved" + saved.write_bytes(original) + selected.unlink() + selected.symlink_to(saved) + elif fault == "oversize": + selected.write_bytes(b" " * 65_537) + else: + actual = os.fstat + + def metadata(fd): + result = actual(fd) + return SimpleNamespace(st_mode=result.st_mode, st_uid=os.geteuid() + 1) + + monkeypatch.setattr(gate.os, "fstat", metadata) + with pytest.raises((gate.Invalid, OSError, ValueError)): + if target == "fixture": + gate.Fixture.read(str(manifest_path)) + else: + gate._launch(args(launch_manifest=str(path)), fixture, config) + if fault != "oversize": + assert selected.read_bytes() == original + + +def native_guard_edges(monkeypatch, config, events): + from contextlib import asynccontextmanager + + import server.settings + from server.auth import tenancy + from server.code.domain.index import repos + from server.core import background, cell, ch, pg, pg_profile + from server.insights.app import provider_command + from server.settings import Settings + + class Client: + connected = False + + async def connect(self): + events.append("native client") + + @asynccontextmanager + async def model_command(): + yield + + native_settings = Settings.model_construct(**config.model_dump(mode="json")) + + async def state(sql): + assert sql == cell._READ + return { + "first": {"is_first": True, "cell": config.cell}, + "mode": "enforced", + "own": {"current_fp": cell.as_value(cell.mine()), "next_fp": None}, + "tenancy": True, + "marked": True, + } + + async def held(sql, org): + assert sql == tenancy._HOLDS and org == "org" + events.append("native held-org guard") + return 1 + + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(cell, "settings", native_settings) + monkeypatch.setattr(tenancy, "settings", native_settings) + monkeypatch.setattr(repos, "settings", native_settings) + monkeypatch.setattr(cell, "_view", cell._View()) + monkeypatch.setattr(cell, "_deployment_id", "") + monkeypatch.setattr(cell, "_code_orgs", None) + monkeypatch.setattr(tenancy, "_held", set()) + monkeypatch.setattr(tenancy, "_elsewhere", {}) + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(pg_profile, "check_configured", AsyncMock()) + monkeypatch.setattr(pg_profile, "deployment_id", AsyncMock(return_value="deployment")) + monkeypatch.setattr(pg, "fetchrow_reset", state) + monkeypatch.setattr(pg, "fetchval", held) + monkeypatch.setattr(provider_command, "model_command", model_command) + monkeypatch.setattr(background, "register_buffer_drainer", lambda *_: None) + monkeypatch.setattr("server.cli._ch.disconnect", AsyncMock()) + + +@pytest.mark.parametrize("credential", ["", "lcr_" + "0" * 128]) +def test_role_pair_enters_the_real_native_guard_without_changing_it(monkeypatch, credential): + from server.cli import _ch + + fixture, config = fixture_config() + events = [] + native_guard_edges(monkeypatch, config, events) + assert fixture.validate(args(), config) == 45500 + config.cell_router_token = credential + + async def work(_): + events.append("native work") + return 17 + + if credential: + with pytest.raises(_ch.OrgNotHeld, match="LOOM_CELL_ROUTER_TOKEN"): + _ch.run_connected(work, orgs=("org",)) + assert events == ["native client"] + else: + assert _ch.run_connected(work, orgs=("org",)) == 17 + assert events == ["native client", "native held-org guard", "native work"] + + +@pytest.mark.parametrize( + "fault", + [ + "", + "api_configuration", + "operator_token", + "arbitrary_setting", + "legacy_field", + "duplicate", + "schema_type", + "pg_port_type", + ], +) +def test_receipt_orchestration_refuses_before_clients_and_keeps_the_real_native_guard( + monkeypatch, tmp_path, capsys, fault +): + import sys + + import server.wiring + + fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest = dataclasses.asdict(fixture) + if fault == "api_configuration": + config.cell_router_token = "lcr_" + "0" * 128 + elif fault == "operator_token": + config.router_operator_token = "lro_" + "1" * 128 + elif fault == "arbitrary_setting": + config.instance_heartbeat_seconds += 1 + manifest["cli_configuration_sha256"] = gate._digest(config.model_dump(mode="json")) + elif fault == "legacy_field": + manifest["configuration_sha256"] = fixture.cli_configuration_sha256 + elif fault == "schema_type": + manifest["schema"] = 2.0 + elif fault == "pg_port_type": + manifest["pg_port"] = float(manifest["pg_port"]) + raw = json.dumps(manifest) + if fault == "duplicate": + raw = raw[:-1] + ', "schema":2}' + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(raw) + output = tmp_path / "output.jsonl" + argv = ["driver"] + for key, value in vars( + args( + launch_manifest=str(launch_path), + fixture_manifest=str(manifest_path), + output=str(output), + ) + ).items(): + argv += ["--" + key.replace("_", "-"), str(value)] + events = [] + native_guard_edges(monkeypatch, config, events) + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) + monkeypatch.setattr( + gate, "_preflight", AsyncMock(side_effect=lambda *_: events.append("readonly")) + ) + monkeypatch.setattr(server.wiring, "wire", lambda: events.append("wire")) + summary = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + + async def run(*_): + events.append("native work") + return summary + + monkeypatch.setattr(gate, "_run", run) + assert gate.main() == int(bool(fault)) + text = capsys.readouterr().out + assert json.loads(text)["verdict"] == ("INVALID" if fault else "PASS") + assert all( + value not in text + for value in [ + args().pg_dsn, + "secret-token", + "test-only-root", + "lcr_" + "0" * 128, + "lro_" + "1" * 128, + ] + ) + if fault: + assert events == [] and not output.exists() + else: + assert events == [ + "readonly", + "wire", + "native client", + "native held-org guard", + "native work", + ] + record = json.loads(output.read_text().splitlines()[0]) + assert record["manifest"]["schema"] == 2 + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + + +def test_unversioned_single_configuration_fixture_is_no_longer_accepted(tmp_path): + legacy = { + "base_url": args().base_url, + "pg_host": "127.0.0.1", + "pg_port": 15790, + "pg_database": "gate_cell", + "ch_host": "127.0.0.1", + "ch_port": 18790, + "ch_database": "gate_cell", + "ch_telemetry_database": "gate_cell_t", + "cell": "gate-cell", + "org": "org", + "namespace": "ns", + "deployment_id": "deployment", + "epoch": 1, + "instance_id": "instance", + "build_commit": "a" * 40, + "source_tree": "b" * 40, + "launch_manifest_sha256": "c" * 64, + "ownership_receipt_sha256": "d" * 64, + "configuration_sha256": "e" * 64, + "background_workers": "off", + "controller_excluded": True, + "pg_internal_port": 5432, + } + path = tmp_path / "legacy.json" + raw = json.dumps(legacy).encode() + path.write_bytes(raw) + with pytest.raises(gate.Invalid): + gate.Fixture.read(str(path)) + assert path.read_bytes() == raw + + +@pytest.mark.parametrize("field", ["pg_port", "ch_port", "epoch"]) +@pytest.mark.parametrize("kind", [bool, float, str]) +def test_fixture_integer_fields_never_coerce_json_types(field, kind): + fixture, config = fixture_config() + with pytest.raises(gate.Invalid, match="fixture field types"): + dataclasses.replace(fixture, **{field: kind(getattr(fixture, field))}).validate( + args(), config + ) diff --git a/tests/unit/test_cell_trigger_gate_artifact_close.py b/tests/unit/test_cell_trigger_gate_artifact_close.py index 3234dc45f5..b403eb4ebb 100644 --- a/tests/unit/test_cell_trigger_gate_artifact_close.py +++ b/tests/unit/test_cell_trigger_gate_artifact_close.py @@ -1,10 +1,13 @@ """Artifact close retains its first flush, sync or file-close failure.""" import asyncio +import dataclasses +import json import pytest from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import fixture_config, receipt_posture @pytest.mark.parametrize("phase", ["flush", "fsync"]) @@ -21,7 +24,14 @@ def test_actual_artifact_close_retains_first_flush_sync_or_close_failure( first = first_kind("flush or sync failed first") if first_kind is not None else None second = second_kind("file close failed second") if second_kind is not None else None artifact = gate.Artifact(str(tmp_path / "raw.jsonl")) - artifact.emit("control") + fixture, config = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) original = artifact.file events = [] @@ -58,3 +68,10 @@ def sync(_fd): assert original.closed and events[-1] == "file close" assert events.count("file close") == 1 assert ("fsync" in events) == (phase != "flush" or first is None) + record = json.loads((tmp_path / "raw.jsonl").read_text()) + assert record["manifest"]["schema"] == 2 + assert record["manifest"]["cli_configuration_sha256"] == gate._digest(config.model_dump()) + assert "configuration_sha256" not in record["manifest"] + assert record["shared_configuration_excludes"] == ["cell_router_token"] + assert record["credential_posture"] == receipt_posture() + assert config.pg_dsn not in json.dumps(record) diff --git a/tests/unit/test_cell_trigger_gate_diagnostics.py b/tests/unit/test_cell_trigger_gate_diagnostics.py index b2f4594ef8..580eb3d733 100644 --- a/tests/unit/test_cell_trigger_gate_diagnostics.py +++ b/tests/unit/test_cell_trigger_gate_diagnostics.py @@ -2,6 +2,7 @@ import argparse import asyncio +import dataclasses import json from unittest.mock import AsyncMock @@ -150,9 +151,12 @@ def test_main_preserves_first_invalid_and_requires_successful_artifact_close( import server.settings import server.wiring from server.cli import _ch - from tests.unit.test_bench_cell_trigger_gate import args, fixture_config + from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, launch_files fixture, config = fixture_config() + fixture, launch_path, _, _, _ = launch_files(tmp_path, fixture, config) + manifest_path = tmp_path / "fixture.json" + manifest_path.write_text(json.dumps(dataclasses.asdict(fixture))) prior = prior_kind("first native work failure") if prior_kind is not None else None output = output_kind("secondary artifact failure") if output_kind is not None else None events = [] @@ -182,15 +186,18 @@ def __getattr__(self, name): argv = ["driver"] for key, value in vars( - args(fixture_manifest="unused", output=str(tmp_path / "raw.jsonl")) + args( + fixture_manifest=str(manifest_path), + launch_manifest=str(launch_path), + output=str(tmp_path / "raw.jsonl"), + ) ).items(): argv += ["--" + key.replace("_", "-"), str(value)] monkeypatch.setattr(sys, "argv", argv) monkeypatch.setattr(server.settings, "settings", config) monkeypatch.setattr(server.wiring, "wire", lambda: None) - monkeypatch.setattr(gate.Fixture, "read", lambda _: fixture) monkeypatch.setattr(gate, "_source", lambda _: {}) - monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_owned_server", lambda _: None) monkeypatch.setattr(gate, "_preflight", AsyncMock()) monkeypatch.setattr(gate, "Artifact", Artifact) diff --git a/tests/unit/test_cell_trigger_gate_first_error.py b/tests/unit/test_cell_trigger_gate_first_error.py index 7a3ccdf41b..5379b0f619 100644 --- a/tests/unit/test_cell_trigger_gate_first_error.py +++ b/tests/unit/test_cell_trigger_gate_first_error.py @@ -1,6 +1,7 @@ """Driver work retains its error through secondary cleanup and artifact failures.""" import asyncio +import dataclasses import json import time from types import SimpleNamespace @@ -11,7 +12,7 @@ from asyncpg.transaction import Transaction from bench import cell_trigger_gate as gate -from tests.unit.test_bench_cell_trigger_gate import args, fixture_config +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config, receipt_posture FIRST = [None, ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] SECOND = [None, OSError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] @@ -46,6 +47,14 @@ def artifact_output(monkeypatch, tmp_path): def configure(errors): artifact = gate.Artifact(str(tmp_path / f"raw-{len(artifacts)}.jsonl")) artifacts.append(artifact) + fixture, _ = fixture_config() + artifact.emit( + "configuration", + manifest=dataclasses.asdict(fixture), + shared_configuration_excludes=["cell_router_token"], + credential_posture=receipt_posture(), + api_configuration_attested=True, + ) original = artifact.file events = [] @@ -73,6 +82,8 @@ def http_edges(monkeypatch): import server.settings fixture, config = fixture_config() + assert fixture.validate(args(), config) == 45500 + assert config.cell_router_token == config.router_operator_token == "" monkeypatch.setattr(server.settings, "settings", config) monkeypatch.setattr(gate, "CALLS", 1) monkeypatch.setattr(gate, "WARMUP", 0) From f485b256edf0918e1171b5222852f92d66431bca Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:49:10 +0000 Subject: [PATCH 097/181] Retain the control deadline during cell configuration updates Signed-off-by: zlareb1 --- server/router/directory.py | 2 +- tests/integration/test_cell_update_run_pg.py | 90 +++++++++++++++++++- tests/unit/test_cell_control_census.py | 2 +- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/server/router/directory.py b/server/router/directory.py index b77f95eebe..db579fa047 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1274,7 +1274,7 @@ async def locked(conn: Any) -> Attaching: raise OperatorRefused(reason) return await _configuration_commit(conn, found, current, pending, new, action) - return await _control_transaction(locked) + return await _control_transaction(locked, seconds=CONTROL_MAX_S) async def _configuration_commit( diff --git a/tests/integration/test_cell_update_run_pg.py b/tests/integration/test_cell_update_run_pg.py index f1d314a12f..db1adc3c33 100644 --- a/tests/integration/test_cell_update_run_pg.py +++ b/tests/integration/test_cell_update_run_pg.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import hashlib import json from argparse import Namespace @@ -12,7 +13,7 @@ from server.cli import cell_control, cell_update_participants, cell_update_pg, cell_update_probe from server.cli.cell_update_run import Run -from server.core import cell, deployment_config +from server.core import cell, deployment_config, pg from server.router import app from server.settings import settings from tests.integration.test_cell_control_fence import control_stores as control_stores @@ -24,6 +25,34 @@ REAL_STORES_CHECK = app.Router._stores_check +async def _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before): + import asyncpg + + observer = await asyncpg.connect(local_dsn) + try: + await pg.init_connection(observer) + local = await cell_update_pg.read(observer, "first") + assert local == before + assert await observer.fetchval("SELECT count(*) FROM loom.tenancy") == 0 + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1)", backend_pid + ) + assert not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1)", backend_pid + ) + finally: + await observer.close() + recorded = await directory_conn.fetchrow("SELECT current_fp,next_fp FROM loom_router.cells") + assert json.loads(recorded["current_fp"]) == value() + assert json.loads(recorded["next_fp"]) == value("new", orgs=("permanent",)) + assert ( + await directory_conn.fetchval( + "SELECT cell FROM loom_router.placements WHERE org=$1", "permanent" + ) + == "first" + ) + + @pytest.fixture async def update_stores(control_stores, local_control, monkeypatch, tmp_path): client, directory_conn, _ = control_stores @@ -102,6 +131,65 @@ def make(action="config", target=None): yield make, local_conn, directory_conn, owner(local_conn), events, local_dsn +@pytest.mark.asyncio +async def test_acceptance_error_rolls_back_tenancy_and_preserves_directory_retry(update_stores): + import asyncpg + + make, local_conn, directory_conn, owned, _, local_dsn = update_stores + backend_pid = local_conn.get_server_pid() + before = await cell_update_pg.read(local_conn, "first") + await local_conn.execute( + "CREATE FUNCTION reject_acceptance() RETURNS trigger LANGUAGE plpgsql AS $$ " + "BEGIN RAISE EXCEPTION 'acceptance refused by test'; END $$; " + "CREATE TRIGGER reject_acceptance BEFORE UPDATE ON loom.cell_self " + "FOR EACH ROW EXECUTE FUNCTION reject_acceptance()" + ) + with pytest.raises(asyncpg.RaiseError, match="acceptance refused by test"): + await make().start(owned) + assert owned.closed and local_conn.is_closed() + await _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before) + + +@pytest.mark.asyncio +async def test_cancelled_acceptance_releases_backend_and_rolls_back_tenancy(update_stores): + import asyncpg + + make, local_conn, directory_conn, owned, _, local_dsn = update_stores + backend_pid = local_conn.get_server_pid() + before = await cell_update_pg.read(local_conn, "first") + blocker = await asyncpg.connect(local_dsn) + transaction = blocker.transaction() + await transaction.start() + task = None + observer = None + try: + observer = await asyncpg.connect(local_dsn) + await blocker.fetchrow("SELECT id FROM loom.cell_self FOR UPDATE") + task = asyncio.create_task(make().start(owned)) + async with asyncio.timeout(4): + while not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event_type='Lock' AND query LIKE 'UPDATE loom.cell_self%')", + backend_pid, + ): + assert not task.done() + await asyncio.sleep(0.005) + task.cancel("acceptance-stop") + with pytest.raises(asyncio.CancelledError, match="acceptance-stop"): + await asyncio.wait_for(task, timeout=4) + assert owned.closed and local_conn.is_closed() + finally: + if task is not None: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await transaction.rollback() + if observer is not None: + await observer.close() + await blocker.close() + await _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before) + + @pytest.mark.asyncio async def test_coordinator_config_and_retry_commit_both_real_stores_once(update_stores): make, local_conn, directory_conn, owned, events, _ = update_stores diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 0f3a129033..efb57fd621 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -478,7 +478,7 @@ def function(path: str, name: str) -> ast.AST: "server/router/directory.py", "update_configuration", [ - "_control_transaction(locked)", + "_control_transaction(locked, seconds=CONTROL_MAX_S)", "control_refusal(found, expected)", "_configuration_commit(conn, found, current, pending, new, action)", ], From c884dd174ea6b07236bf886ff7129e4ef15c6989 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 04:51:38 +0000 Subject: [PATCH 098/181] Preserve native errors through nonce cleanup Signed-off-by: zlareb1 --- server/cli/cell.py | 3 +- server/cli/cell_attach.py | 15 +- server/cli/cell_prepare.py | 6 +- .../unit/test_cell_native_http_first_error.py | 6 +- .../test_cell_native_nonce_first_error.py | 269 ++++++++++++++++++ 5 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_cell_native_nonce_first_error.py diff --git a/server/cli/cell.py b/server/cli/cell.py index e43a5a4d07..fb1b04db95 100644 --- a/server/cli/cell.py +++ b/server/cli/cell.py @@ -349,7 +349,7 @@ def _cmd_remove(args: argparse.Namespace) -> int: print(f"loom cell remove: refusing: {reason}", file=sys.stderr) return 1 try: - answer = asyncio.run(_remove_locked(args, router, row)) + answer = asyncio.run(_remove_locked(args, router)) except Exception as e: # noqa: BLE001 — an unknown control outcome stops the run print(f"loom cell remove: {type(e).__name__}: {e}; stop and rerun", file=sys.stderr) return 1 @@ -360,7 +360,6 @@ def _cmd_remove(args: argparse.Namespace) -> int: async def _remove_locked( args: argparse.Namespace, router: httpx.URL, - row: dict[str, Any], transport: httpx.AsyncBaseTransport | None = None, ) -> dict[str, Any]: from server.cli import cell_control diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 8fd6e7ebee..821a77448b 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -376,7 +376,11 @@ async def begin(self) -> None: await _write_nonces(self.ch, nonces) self.row = {**self.row, **await self.mutate("/attach/begin", body)} self.begun_keys_fp = self.keys_fp - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) # -- the scans --------------------------------------------------------------------- @@ -542,11 +546,16 @@ async def serving(self) -> dict[str, Any]: nonces = _new_nonces() try: await _write_nonces(self.ch, nonces) - return await self.mutate( + answer = await self.mutate( "/attach/serving", {"deployment_id": self.row["deployment_id"], "nonces": nonces} ) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(self.ch, nonces) + raise + else: await _delete_nonces(self.ch, nonces) + return answer # -- the run ----------------------------------------------------------------------- diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index 4310b4c9bb..517a461840 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -513,7 +513,11 @@ async def _prepare_locked( try: await _write_nonces(ch, body["nonces"]) answer = await _register(args, router, body, transport) - finally: + except BaseException: # Nonce cleanup preserves the first failure or cancellation. + with contextlib.suppress(BaseException): + await _delete_nonces(ch, body["nonces"]) + raise + else: await _delete_nonces(ch, body["nonces"]) # The processes may have changed while the probe was retried: judge the ones live now. beats = await _read_beats(ch) diff --git a/tests/unit/test_cell_native_http_first_error.py b/tests/unit/test_cell_native_http_first_error.py index 4634e7d69c..c9b61bd146 100644 --- a/tests/unit/test_cell_native_http_first_error.py +++ b/tests/unit/test_cell_native_http_first_error.py @@ -180,11 +180,7 @@ async def test_required_client_exit_failure_cannot_report_success( async def test_successful_client_neighbor(exchange, boundary): kind, result = await observed(lambda: invoke(boundary)) assert kind == "value" - expected = ( - None - if boundary == "placement" - else ("dep-eu-1" if boundary == "health" else exchange.answer) - ) + expected = "dep-eu-1" if boundary == "health" else exchange.answer assert result == expected assert exchange.events == ["construct", "enter", "request", "exit-call", "exit-await"] diff --git a/tests/unit/test_cell_native_nonce_first_error.py b/tests/unit/test_cell_native_nonce_first_error.py new file mode 100644 index 0000000000..94254c3d96 --- /dev/null +++ b/tests/unit/test_cell_native_nonce_first_error.py @@ -0,0 +1,269 @@ +"""Native callers retain their first outcome while deleting their probe nonces.""" + +from __future__ import annotations + +import argparse +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from server.cli import cell_attach, cell_control, cell_prepare +from server.core import leases, object_store, pg +from server.settings import settings +from tests.unit import test_cell_native_http_first_error as native_http +from tests.unit.test_cell_attach import FakeConn +from tests.unit.test_cell_control import snapshot + +exchange = native_http.exchange +observed = native_http.observed +FIRST_ERRORS = native_http.FIRST_ERRORS +CALLERS = ("prepare", "begin", "serving") +FATAL_ERRORS = (asyncio.CancelledError, SystemExit, KeyboardInterrupt) +CLEANUP_PHASES = ("postgres", "clickhouse", "bucket") +URL = httpx.URL("http://router.test") + + +@pytest.fixture +def nonce_edges(monkeypatch, exchange): + state = SimpleNamespace( + secondary=None, + phase="postgres", + write_first=False, + events=[], + normalized=[], + cleanup_entered=asyncio.Event(), + cleanup_block=False, + ) + nonces = {"postgres": "pg", "clickhouse": "ch", "bucket": "bucket"} + + async def deletion(phase): + state.events.append(phase) + if phase == state.phase: + state.cleanup_entered.set() + if state.cleanup_block: + await asyncio.Future() + if state.secondary is not None: + raise state.secondary + + async def postgres(*args): + await deletion("postgres") + + async def clickhouse(*args): + assert args[0] == leases.LOCK_RELEASE + await deletion("clickhouse") + + async def read_object(location): + return b"bucket" + + async def bucket(*args): + await deletion("bucket") + + async def write(*args): + if state.write_first: + raise exchange.first + + monkeypatch.setattr(pg, "execute", postgres) + monkeypatch.setattr(object_store, "root_prefix", lambda: "") + monkeypatch.setattr(object_store, "read", read_object) + monkeypatch.setattr(object_store, "delete", bucket) + monkeypatch.setattr(cell_prepare, "_new_nonces", lambda: nonces.copy()) + monkeypatch.setattr(cell_prepare, "_write_nonces", write) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "bucket-test") + state.ch = SimpleNamespace(execute=clickhouse) + return state + + +@pytest.fixture +def native_call(monkeypatch, nonce_edges, exchange): + state = nonce_edges + row = {**snapshot(), "is_first": False, "state": "attaching", "mode": "enforced"} + exchange.answer = row.copy() + conn = FakeConn() + rows = SimpleNamespace(deployment_id="dep", first=None, tenancy_rows=0) + monkeypatch.setattr(cell_prepare, "_read_rows", AsyncMock(return_value=rows)) + monkeypatch.setattr(cell_prepare, "_read_beats", AsyncMock(return_value=[])) + monkeypatch.setattr(cell_prepare, "_beat_fields", lambda *args: [{}]) + monkeypatch.setattr(cell_prepare, "_state_rows", lambda rows: 0) + monkeypatch.setattr(cell_prepare, "_prefix_objects", AsyncMock(return_value=0)) + monkeypatch.setattr(cell_prepare, "_answer_problem", lambda *args: "") + monkeypatch.setattr(cell_prepare, "_own_rows", lambda *args: ({}, None)) + state.write_rows = AsyncMock(return_value=[]) + monkeypatch.setattr(cell_prepare, "_write_rows", state.write_rows) + monkeypatch.setattr(cell_control, "claim", AsyncMock(side_effect=lambda conn, call, row: row)) + args = argparse.Namespace( + cell="first", + wait=0, + host="first.example.test", + credential_sha256="hash", + probe_url="", + ch_allow_user=[], + ch_allow_user_directory=[], + ch_allow_object=[], + ch_allow_replica=[], + pg_allow_role=[], + ) + + async def mutate(path, body): + return await cell_control.request(URL, "first", None, path, body) + + run = SimpleNamespace( + ch=state.ch, args=args, row=row, own_fp=({}, None), keys_fp="key", mutate=mutate + ) + + async def invoke(caller, owned): + owner = cell_control.Owner(exit_s=0.5) + direct = SimpleNamespace( + conn=conn, transaction=lambda work: work(), call=lambda work, **kw: work() + ) + + async def work(): + if caller == "prepare": + return await cell_prepare._prepare_locked( + state.ch, args, URL, None, owner if owned else direct + ) + return await getattr(cell_attach.Run, caller)(run) + + async def opening(): + owner.conn = conn + + return await owner.run(opening, work, opening_s=0.5) if owned else await work() + + state.conn = conn + state.invoke = invoke + return state + + +@pytest.fixture(autouse=True) +def selected_normalization(monkeypatch, nonce_edges): + original_prepare = cell_prepare.Refused + original_control = cell_control._refused + + class PrepareRefused(original_prepare): + def __init__(self, message): + super().__init__(message) + nonce_edges.normalized.append(self) + + def control_refused(message): + selected = original_control(message) + nonce_edges.normalized.append(selected) + return selected + + monkeypatch.setattr(cell_prepare, "Refused", PrepareRefused) + monkeypatch.setattr(cell_control, "_refused", control_refused) + + +def assert_first(state, exchange, caller, first_kind, owned, outcome): + kind, result = outcome + assert kind == "error" + normalized = not state.write_first and ( + (caller == "prepare" and issubclass(first_kind, httpx.HTTPError)) + or (caller != "prepare" and issubclass(first_kind, (httpx.HTTPError, TimeoutError))) + ) + if normalized: + assert state.normalized and result is state.normalized[0] + elif owned and first_kind is asyncio.CancelledError: + # asyncio reconstructs cancellation when Owner retrieves its child Task. + assert isinstance(result, asyncio.CancelledError) + else: + assert result is exchange.first + assert state.write_rows.await_count == 0 + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("first_kind", FIRST_ERRORS) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +@pytest.mark.parametrize("http_exit_failure", (False, True)) +async def test_http_first_outcome_survives_native_nonce_cleanup( + native_call, exchange, caller, owned, first_kind, cleanup_kind, phase, http_exit_failure +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-second") if cleanup_kind is not None else None + exchange.first = first_kind("request-first") + exchange.second = KeyboardInterrupt("http-exit-second") if http_exit_failure else None + assert_first( + state, + exchange, + caller, + first_kind, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert phase in state.events and len(exchange.exit_args) == 3 + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", (*FATAL_ERRORS, ValueError, None)) +@pytest.mark.parametrize("phase", CLEANUP_PHASES) +async def test_standalone_nonce_cleanup_keeps_cancellation_and_success_neighbors( + native_call, exchange, caller, owned, cleanup_kind, phase +): + state = native_call + state.phase = phase + state.secondary = cleanup_kind("nonce-only") if cleanup_kind is not None else None + kind, result = await observed(lambda: state.invoke(caller, owned)) + if cleanup_kind in FATAL_ERRORS: + assert kind == "error" + if owned and cleanup_kind is asyncio.CancelledError: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is state.secondary + assert state.write_rows.await_count == 0 + else: + assert kind == "value" + assert result == ( + 0 if caller == "prepare" else None if caller == "begin" else exchange.answer + ) + assert state.events == list(CLEANUP_PHASES) + assert state.write_rows.await_count == int(caller == "prepare") + assert state.conn.executed.count("CLOSE") == int(owned) + + +@pytest.mark.parametrize("caller", CALLERS) +@pytest.mark.parametrize("owned", (False, True)) +@pytest.mark.parametrize("cleanup_kind", FATAL_ERRORS) +async def test_nonce_write_failure_stays_first(native_call, exchange, caller, owned, cleanup_kind): + state = native_call + state.write_first = True + state.secondary = cleanup_kind("nonce-second") + exchange.first = ValueError("write-first") + assert_first( + state, + exchange, + caller, + ValueError, + owned, + await observed(lambda: state.invoke(caller, owned)), + ) + assert exchange.events == [] and state.events == ["postgres"] + + +@pytest.mark.parametrize("caller", CALLERS) +async def test_repeated_cancellation_keeps_first_stop_through_nonce_deletion( + native_call, exchange, caller +): + state = native_call + state.cleanup_block = True + exchange.block = True + task = asyncio.create_task(observed(lambda: state.invoke(caller, False))) + try: + await asyncio.wait_for(exchange.entered.wait(), 1) + task.cancel("first-stop") + await asyncio.wait_for(state.cleanup_entered.wait(), 1) + task.cancel("second-stop") + kind, result = await task + assert kind == "error" and result is exchange.first + assert isinstance(result, asyncio.CancelledError) and result.args == ("first-stop",) + assert state.write_rows.await_count == 0 + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) From 0a94de8d45f42697f8caa1f62736b3ca7c005f26 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 05:14:56 +0000 Subject: [PATCH 099/181] Native commands preserve their first failure through cleanup Connection and participant cleanup preserve entry and work errors. Successful commands require stopped producers, drained writes, flushed buffers and a positive retirement acknowledgement. Signed-off-by: zlareb1 --- server/cli/_ch.py | 22 +- server/insights/app/provider_command.py | 102 ++++-- tests/unit/test_cli_native_first_error.py | 415 ++++++++++++++++++++++ tests/unit/test_provider_command.py | 7 +- 4 files changed, 516 insertions(+), 30 deletions(-) create mode 100644 tests/unit/test_cli_native_first_error.py diff --git a/server/cli/_ch.py b/server/cli/_ch.py index c67479b965..7772fbdb1a 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -23,6 +23,7 @@ import logging import sys from collections.abc import Awaitable, Callable, Iterable +from contextlib import suppress from dataclasses import dataclass from typing import TYPE_CHECKING, Literal @@ -299,7 +300,8 @@ async def connect_declared( await check_configured(ch) await check_acceptance(orgs=named, org_free=org_free) except BaseException: - await disconnect(ch) + with suppress(BaseException): + await disconnect(ch) raise @@ -338,7 +340,8 @@ async def connect_verified(ch: CHClient) -> None: try: await check_configured(ch) except BaseException: - await disconnect(ch) + with suppress(BaseException): + await disconnect(ch) raise from server import boot from server.core.background import register_buffer_drainer @@ -362,7 +365,11 @@ async def disconnect(ch: CHClient) -> None: pg.expect_deployment(None) try: await pg.close_pool() - finally: + except BaseException: + with suppress(BaseException): + await ch.disconnect() + raise + else: await ch.disconnect() @@ -389,8 +396,13 @@ async def _go() -> T: try: async with model_command(): await check_acceptance(orgs=named, org_free=org_free) - return await run(ch) - finally: + result = await run(ch) + except BaseException: + with suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) + return result return asyncio.run(_go()) diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index b89c418e17..3b9f7d8a4f 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -3,6 +3,7 @@ import asyncio import contextlib import logging +import sys from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager @@ -34,12 +35,63 @@ async def _adopt_overrides(*, failing: bool = False) -> bool: async def _keep_overrides( loaded: bool, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep -) -> None: +) -> SystemExit | KeyboardInterrupt | None: """A long command (a rechunk, a re-embed) follows an override set while it runs.""" set_query_tag("cli.settings_overrides") - while True: - await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) - loaded = await _adopt_overrides(failing=not loaded) + try: + while True: + await sleep(max(1.0, float(settings.runtime_overrides_refresh_seconds))) + loaded = await _adopt_overrides(failing=not loaded) + except (SystemExit, KeyboardInterrupt) as error: + # A task's fatal exception must reach its owner before the loop stops. + return error + + +async def _finish_participant() -> None: + gate = retirement.get_gate() + gate.close() + from server.memory.app import embedder_calibration + + first: BaseException | None = None + stopped = False + try: + embedder_calibration.stop_refresh_loop() + stopped = True + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + first = error + producers = tuple(gate.producers) + for producer in producers: + producer.cancel() + try: + if producers: + _, pending = await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + if pending: + raise RuntimeError("Native command producers did not stop") + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + stopped = False + for producer in producers: + if producer.done() and not producer.cancelled(): + try: + producer.result() + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + if stopped: + gate.stopped() + try: + if not await drain_task_sets(timeout_s=COMMAND_DRAIN_S): + raise RuntimeError("Native command deferred writes did not drain") + if not await flush_registered_buffers(): + raise RuntimeError("Native command buffers did not flush") + if await retire_process() is not True: + raise RuntimeError("Native command retirement was not certified") + except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if first is None: + first = error + if first is not None: + raise first @asynccontextmanager @@ -50,27 +102,19 @@ async def participant_command() -> AsyncIterator[None]: try: async with retirement.get_gate().admit(): yield - finally: - gate = retirement.get_gate() - gate.close() - from server.memory.app import embedder_calibration - - embedder_calibration.stop_refresh_loop() - producers = tuple(gate.producers) - for producer in producers: - producer.cancel() - if producers: - await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) - gate.stopped() - if await drain_task_sets(timeout_s=COMMAND_DRAIN_S) and await flush_registered_buffers(): - await retire_process() + except BaseException: + with contextlib.suppress(BaseException): + await _finish_participant() + raise + else: + await _finish_participant() @asynccontextmanager async def model_command() -> AsyncIterator[None]: """The permanent lifecycle also holds the reversible model-work admission.""" async with participant_command(): - keeper: asyncio.Task[None] | None = None + keeper: asyncio.Task[SystemExit | KeyboardInterrupt | None] | None = None try: await load_active_configuration() loaded = await _adopt_overrides() @@ -80,7 +124,21 @@ async def model_command() -> AsyncIterator[None]: async with get_model_work_gate().admit(): yield finally: + first = sys.exception() if keeper is not None: - keeper.cancel() - with contextlib.suppress(asyncio.CancelledError): - await keeper + owner = asyncio.current_task() + assert owner is not None + cancelling = owner.cancelling() + requested = keeper.cancel() + try: + try: + fatal = await keeper + except asyncio.CancelledError: + if not requested or owner.cancelling() != cancelling: + raise + else: + if fatal is not None: + raise fatal + except BaseException: + if first is None: + raise diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py new file mode 100644 index 0000000000..35f71b60fb --- /dev/null +++ b/tests/unit/test_cli_native_first_error.py @@ -0,0 +1,415 @@ +"""The native command lifetime preserves its first failure through owned cleanup.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import background, retirement +from server.core.model_work_gate import ModelWorkGate +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command + + +@pytest.fixture +def native(monkeypatch): + from server import boot + from server.core import ch, pg, pg_profile + from server.memory.app import embedder_calibration + + events = [] + gate = retirement.get_gate() + client = SimpleNamespace(connected=False) + errors = {} + model_gate = ModelWorkGate() + + def event(name): + events.append(name) + if name in errors: + raise errors[name] + + async def connect(): + event("connect") + client.connected = True + + async def close_ch(): + client.connected = False + event("ch") + + async def close_pg(): + event("pg") + + async def verify(_): + assert not gate.enrolled + event("verify") + + async def enroll(): + event("enroll") + gate.enrolled = True + + async def acceptance(_): + assert gate.enrolled and gate.active and not gate.closed + event("acceptance") + + async def configuration(): + event("configuration") + + async def flush(): + event("flush") + + async def certificate(proof): + assert gate.quiescent and gate.enrolled and client.connected + assert proof.version > proof.after + assert events[-1] == "flush" + event("certificate") + + client.connect, client.disconnect = connect, close_ch + monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(pg, "close_pool", close_pg) + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(_ch, "check_orgs", acceptance) + monkeypatch.setattr(command, "register_process", enroll) + monkeypatch.setattr(command, "load_active_configuration", configuration) + monkeypatch.setattr(command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(command, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(barrier, "get_model_work_gate", lambda: model_gate) + monkeypatch.setattr(background, "_registered_tasks", set()) + monkeypatch.setattr(background, "_buffer_drainers", {}) + monkeypatch.setattr(boot, "drain_writes", flush) + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=False) + ) + monkeypatch.setattr(barrier, "_server_side_settled", AsyncMock(return_value=True)) + monkeypatch.setattr(barrier, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(barrier, "record_retirement", certificate) + monkeypatch.setattr(embedder_calibration, "stop_refresh_loop", lambda: event("stop")) + return SimpleNamespace( + events=events, errors=errors, client=client, gate=gate, model_gate=model_gate, event=event + ) + + +FIRST = [ValueError, asyncio.CancelledError, SystemExit, KeyboardInterrupt] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("entry", ["verify", "acceptance", "configuration", "work"]) +@pytest.mark.parametrize("secondary", ["pg", "ch", "certificate"]) +def test_native_first_entry_or_work_error_survives_all_close_boundaries( + native, first_class, entry, secondary +): + first = first_class("first") + native.errors[entry] = first + native.errors[secondary] = OSError("secondary") + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.model_gate.active == native.gate.active == 0 + if entry != "work": + assert "work" not in native.events + if entry != "verify": + assert "stop" in native.events + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("connector", ["connect_checked", "connect_verified"]) +@pytest.mark.asyncio +async def test_connect_entry_preserves_refusal_and_attempts_both_closes( + native, first_class, connector +): + first = first_class("pairing refusal") + native.errors.update(verify=first, pg=OSError("pg"), ch=RuntimeError("ch")) + call = getattr(_ch, connector) + with pytest.raises(first_class) as caught: + if connector == "connect_checked": + await call(native.client, orgs=("o",)) + else: + await call(native.client) + assert caught.value is first + assert native.events == ["connect", "verify", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.asyncio +async def test_disconnect_preserves_first_pg_failure_and_still_closes_ch(native, first_class): + first = first_class("pg close") + native.errors.update(pg=first, ch=OSError("ch close")) + with pytest.raises(first_class) as caught: + await _ch.disconnect(native.client) + assert caught.value is first + assert native.events == ["pg", "ch"] + + +@pytest.mark.parametrize("cleanup", ["stop", "flush", "certificate", "pg", "ch"]) +@pytest.mark.parametrize("first_class", FIRST) +def test_required_native_cleanup_error_after_success_is_fatal(native, cleanup, first_class): + first = first_class(cleanup) + native.errors[cleanup] = first + + async def work(_): + native.event("work") + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + + +def test_first_required_cleanup_error_wins_and_all_safe_cleanup_runs(native, monkeypatch): + first = OSError("keeper") + + async def keeper(_): + raise first + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=ValueError("retirement"), pg=RuntimeError("pg")) + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert "stop" in native.events and "flush" in native.events + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_driver_restore_failure_remains_first_when_keeper_and_native_close_fail( + native, monkeypatch, first_class +): + first = first_class("driver work") + + async def keeper(_): + raise OSError("keeper") + + async def work(_): + await asyncio.sleep(0) + try: + raise first + finally: + native.events.append("restore") + + monkeypatch.setattr(command, "_keep_overrides", keeper) + native.errors.update(certificate=OSError("certificate"), pg=OSError("pg"), ch=OSError("ch")) + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events.index("restore") < native.events.index("stop") + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("phase", ["drain", "flush", "retire"]) +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_unacknowledged_cleanup_refuses_without_early_certification( + native, monkeypatch, phase, first_class +): + calls = [] + first = first_class("work") if first_class else None + + async def unacknowledged(**_): + calls.append(phase) + return False + + target = { + "drain": "drain_task_sets", + "flush": "flush_registered_buffers", + "retire": "retire_process", + } + monkeypatch.setattr(command, target[phase], unacknowledged) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or RuntimeError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + assert calls == [phase] + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +def test_actual_deferred_descendant_and_role_producer_end_before_certificate(native): + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + native.events.append("producer-ended") + + native.gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + + async def descendant(): + await asyncio.sleep(0) + native.events.append("descendant") + + async def deferred(): + await asyncio.sleep(0) + background.create_deferred_task(descendant()) + + background.create_deferred_task(deferred()) + return 7 + + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.index("producer-ended") < native.events.index("certificate") + assert native.events.index("descendant") < native.events.index("certificate") + assert native.gate.quiescent and native.model_gate.active == 0 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_override_keeper_returns_fatal_to_owner_before_loop_shutdown(first_class): + first = first_class("keeper fatal") + + async def sleep(_): + raise first + + assert await command._keep_overrides(True, sleep=sleep) is first + + +@pytest.mark.parametrize("first_class", [SystemExit, KeyboardInterrupt]) +def test_real_override_keeper_fatal_is_retrieved_before_native_cleanup( + native, monkeypatch, first_class +): + first = first_class("keeper fatal") + actual = command._keep_overrides + + async def sleep(_): + raise first + + monkeypatch.setattr(command, "_keep_overrides", lambda loaded: actual(loaded, sleep=sleep)) + + async def work(_): + await asyncio.sleep(0) + return 7 + + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert native.gate.quiescent + + +def test_clean_native_success_has_one_certified_exit_and_no_keeper_left(native, monkeypatch): + keepers = [] + actual = command._keep_overrides + + async def keeper(loaded): + keepers.append(asyncio.current_task()) + return await actual(loaded) + + async def work(_): + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + assert _ch.run_connected(work, orgs=("o",)) == 7 + assert native.events.count("certificate") == 1 + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_caller_cancellation_during_keeper_stop_is_not_swallowed(native, monkeypatch): + owner = [] + keepers = [] + + async def keeper(_): + keepers.append(asyncio.current_task()) + try: + await asyncio.Event().wait() + finally: + owner[0].cancel("cleanup cancellation") + await asyncio.sleep(0) + + async def work(_): + owner.append(asyncio.current_task()) + await asyncio.sleep(0) + return 7 + + monkeypatch.setattr(command, "_keep_overrides", keeper) + with pytest.raises(asyncio.CancelledError, match="cleanup cancellation"): + _ch.run_connected(work, orgs=("o",)) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert keepers and all(task.done() for task in keepers) + + +def test_unfinished_role_producer_prevents_success_and_certificate(native, monkeypatch): + producers = [] + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0) + + async def work(_): + async def producer(): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await asyncio.Event().wait() + + task = native.gate.track(asyncio.create_task(producer())) + producers.append(task) + await asyncio.sleep(0) + return 7 + + with pytest.raises(RuntimeError, match="producers did not stop"): + _ch.run_connected(work, orgs=("o",)) + assert "certificate" not in native.events + assert not native.gate.producers_stopped + assert producers and all(task.done() for task in producers) + + +def test_first_role_producer_failure_is_retrieved_and_does_not_skip_safe_drain(native, monkeypatch): + first = OSError("producer failed") + producers = [] + + async def work(_): + async def producer(error): + raise error + + # Hold completed producer handles until the composition retrieves both errors. + for error in [first, first]: + task = asyncio.create_task(producer(error)) + native.gate.producers.add(task) + producers.append(task) + await asyncio.sleep(0) + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in producers) + + +def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): + stopped = [] + monkeypatch.setattr( + barrier, + "_HEARTBEAT", + SimpleNamespace(signal_stop=lambda: stopped.append(True), running=True), + ) + + async def work(_): + return 7 + + with pytest.raises(RuntimeError, match="retirement was not certified"): + _ch.run_connected(work, orgs=("o",)) + assert stopped == [True] + assert "certificate" not in native.events + assert native.gate.enrolled and native.gate.quiescent + assert native.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index c6ba785ab4..2a1d22da2d 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -16,7 +16,7 @@ def setup(monkeypatch): gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) - monkeypatch.setattr(command, "retire_process", AsyncMock()) + monkeypatch.setattr(command, "retire_process", AsyncMock(return_value=True)) monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(command, "flush_registered_buffers", AsyncMock(return_value=True)) @@ -46,8 +46,9 @@ async def test_pending_transition_never_runs_command(setup): @pytest.mark.asyncio async def test_undrained_descendants_keep_participant_registered(setup, monkeypatch): monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=False)) - async with command.model_command(): - pass + with pytest.raises(RuntimeError, match="deferred writes did not drain"): + async with command.model_command(): + pass command.retire_process.assert_not_awaited() command.flush_registered_buffers.assert_not_awaited() From c11f8eaf2d5306758f0af35b76e4112785b210d1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 13:50:40 +0000 Subject: [PATCH 100/181] Native commands retain failures through participant cleanup Cleanup preserves the first failure and retries cancellation only when the owning task receives another cancellation request. Native wrappers and timing receipts retain the same completion boundary. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 9 + server/cli/_ch.py | 7 +- server/cli/analytics_export.py | 7 +- server/cli/code_run.py | 18 +- server/cli/doctor.py | 14 +- server/cli/doctor_cell.py | 15 +- server/cli/import_brain.py | 7 +- server/cli/import_clickgap.py | 7 +- server/cli/import_recall.py | 58 ++-- server/cli/maintenance.py | 7 +- server/cli/sql_access.py | 7 +- server/core/retirement.py | 12 +- server/insights/app/provider_barrier.py | 37 +- server/insights/app/provider_command.py | 57 ++-- .../unit/test_cell_trigger_gate_json_codec.py | 201 +++++++++++ tests/unit/test_cli_native_completion.py | 252 ++++++++++++++ tests/unit/test_cli_native_first_error.py | 15 +- tests/unit/test_cli_native_wrappers.py | 316 ++++++++++++++++++ tests/unit/test_provider_command.py | 12 +- 19 files changed, 975 insertions(+), 83 deletions(-) create mode 100644 tests/unit/test_cell_trigger_gate_json_codec.py create mode 100644 tests/unit/test_cli_native_completion.py create mode 100644 tests/unit/test_cli_native_wrappers.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 19db375e4b..3753ab4f90 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1210,6 +1210,15 @@ def _compare( and left == right, phase + " vector", ) + elif field_name == "value_json": + try: + # PostgreSQL keeps JSON text; the native ClickHouse decoder returns its value. + left = json.loads(left) if isinstance(left, str) else left + right = json.loads(right) if isinstance(right, str) else right + same = sha256_value(left) == sha256_value(right) + except (ValueError, TypeError) as error: + raise Invalid(phase + " " + field_name) from error + _require(same, phase + " " + field_name) else: _require(_normal(left) == _normal(right), phase + " " + field_name) diff --git a/server/cli/_ch.py b/server/cli/_ch.py index 7772fbdb1a..bca6dadbaa 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -228,7 +228,12 @@ async def check_setup_stores() -> None: pg.expect_deployment((str(recorded["deployment_id"]), int(recorded["epoch"]))) await check_org_free(writes("ClickHouse users and grants, no tenant rows")) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a close failure must not replace the store-check refusal + if first is None: + raise async def check_orgs(orgs: Iterable[str]) -> None: diff --git a/server/cli/analytics_export.py b/server/cli/analytics_export.py index 5f5abf8bb3..63bd9732dd 100644 --- a/server/cli/analytics_export.py +++ b/server/cli/analytics_export.py @@ -105,7 +105,12 @@ async def _go() -> list[Exported]: day, prefix=args.prefix, put=put, name_key=name_key, ch=ch ) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise for e in asyncio.run(_go()): # A kept object's size is the earlier run's, which this run never read. diff --git a/server/cli/code_run.py b/server/cli/code_run.py index 1074951fa1..5c11a94f60 100644 --- a/server/cli/code_run.py +++ b/server/cli/code_run.py @@ -134,13 +134,23 @@ async def _then_disconnect(coro: asyncio.Future[int] | Coroutine[Any, Any, int]) # Some native commands close their own client. Reopen it for # buffered-write drain/retirement before leaving enrollment. # The subcommand's own connect gated its writes; a drain checks nothing new. - await connect_declared( - get_client(), org_free=reads("drains the writes its own connect gated") - ) + first = sys.exception() + try: + await connect_declared( + get_client(), org_free=reads("drains the writes its own connect gated") + ) + except BaseException: # keep the subcommand failure if drain reconnect also fails + if first is None: + raise finally: + first = sys.exception() if not started and asyncio.iscoroutine(coro): coro.close() - await disconnect(get_client()) + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the subcommand failure + if first is None: + raise def _run(name: str, coro: asyncio.Future[int] | Coroutine[Any, Any, int]) -> int: diff --git a/server/cli/doctor.py b/server/cli/doctor.py index 29a9ddabf6..b4814b7978 100644 --- a/server/cli/doctor.py +++ b/server/cli/doctor.py @@ -482,7 +482,12 @@ async def read() -> tuple[Any, ...] | None: await tier_health.refresh_failing() return tier_health.problem_rows() finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise _read_code_registry() # the self-service entries too; a no-op once a read succeeded try: @@ -509,7 +514,12 @@ async def read() -> None: await connect_declared(client, org_free=reads("reads")) await repo_registry.refresh_db_repos(force=True) finally: - await disconnect(client) + first = sys.exception() + try: + await disconnect(client) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise try: asyncio.run(asyncio.wait_for(read(), timeout=5)) diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index e8a5fc83c1..79fbebb5af 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import sys from typing import Any from server.cli._ch import reads @@ -216,7 +217,12 @@ async def _go() -> list[tuple[str, str, str]]: except Exception as e: # noqa: BLE001 — a ClickHouse that does not answer is named return [("cell_state", WARN, f"not read: {_reason(e)}")] finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise return asyncio.run(_go()) @@ -341,7 +347,12 @@ async def _go() -> list[tuple[str, str, str]]: except Exception as e: # noqa: BLE001 — an unread store is named, not a failed run return [("cell_stray_orgs", WARN, f"not read: {_reason(e)}")] finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # keep the diagnostic failure when close also fails + if first is None: + raise with statement_timeout(budget_s): return asyncio.run(_go()) diff --git a/server/cli/import_brain.py b/server/cli/import_brain.py index 8247dcf699..10bead6dba 100644 --- a/server/cli/import_brain.py +++ b/server/cli/import_brain.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import contextlib import logging from typing import Any @@ -157,7 +158,11 @@ async def _run() -> int: except Exception as e: # noqa: BLE001 — one document must not sink the rest fail += 1 log.warning("import-brain: knowledge.add %s failed: %s", uri, e) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"import-brain: wrote {ok} item(s), {fail} failed") return 1 if fail and ok == 0 else 0 diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index 983ce671cd..cc44181142 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -901,7 +901,12 @@ async def _run_then_disconnect() -> int: async with command_stack: return await _run() finally: - await disconnect(get_client()) + first = sys.exception() + try: + await disconnect(get_client()) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise from contextlib import AsyncExitStack diff --git a/server/cli/import_recall.py b/server/cli/import_recall.py index b24c9371b5..6e49c95b84 100644 --- a/server/cli/import_recall.py +++ b/server/cli/import_recall.py @@ -731,7 +731,11 @@ async def _run() -> int: f" ! write failed ({w['op']} {ident}): {e}", file=sys.stderr, ) - finally: + except BaseException: # an importer failure must survive its writer close + with contextlib.suppress(BaseException): + await closer() + raise + else: await closer() print(f"✓ import complete: {ok} writes ok, {failed} failed") @@ -755,6 +759,27 @@ def _publication_pending(r: Any) -> bool: return isinstance(detail, dict) and detail.get("code") == PublicationPending.code +def _make_http_writer( + args: argparse.Namespace, base_url: str, *, timeout_s: float +) -> tuple[Any, Any]: + import httpx + + token, source = resolve_token(args.token) + if refused_for_agent(source): + raise SystemExit(2) + headers = {"Authorization": f"Bearer {token}"} if token else {} + client = httpx.AsyncClient(timeout=timeout_s, headers=headers) + base = base_url.rstrip("/") + + async def _http_writer(op: str, payload: dict[str, Any]) -> None: + r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) + if _publication_pending(r): + return # committed; the server's delivery worker publishes it + r.raise_for_status() + + return _http_writer, client.aclose + + async def _make_writer( args: argparse.Namespace, inproc_ops: Callable[[], _OpHandlers], *, timeout_s: float ) -> tuple[Any, Any]: @@ -764,22 +789,7 @@ async def _make_writer( its handler) on a connected ClickHouse client.""" base_url = getattr(args, "base_url", None) if base_url: - import httpx - - token, source = resolve_token(args.token) - if refused_for_agent(source): - raise SystemExit(2) - headers = {"Authorization": f"Bearer {token}"} if token else {} - client = httpx.AsyncClient(timeout=timeout_s, headers=headers) - base = base_url.rstrip("/") - - async def _http_writer(op: str, payload: dict[str, Any]) -> None: - r = await client.post(f"{base}/v1/{op}", json=_recall_jsonable(payload)) - if _publication_pending(r): - return # committed; the server's delivery worker publishes it - r.raise_for_status() - - return _http_writer, client.aclose + return _make_http_writer(args, base_url, timeout_s=timeout_s) from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client @@ -798,11 +808,11 @@ async def _http_writer(op: str, payload: dict[str, Any]) -> None: await command.__aenter__() entered = True await check_acceptance(orgs=(args.org,)) - except BaseException: - try: - if entered: + except BaseException: # entry failure must survive participant and connection cleanup + if entered: + with contextlib.suppress(BaseException): await command.__aexit__(None, None, None) - finally: + with contextlib.suppress(BaseException): await disconnect(ch) raise @@ -816,7 +826,11 @@ async def _inproc_writer(op: str, payload: dict[str, Any]) -> None: async def close() -> None: try: await command.__aexit__(None, None, None) - finally: + except BaseException: # participant exit failure must survive connection cleanup + with contextlib.suppress(BaseException): + await disconnect(ch) + raise + else: await disconnect(ch) return _inproc_writer, close diff --git a/server/cli/maintenance.py b/server/cli/maintenance.py index 9a6cdb2437..9e9a3bbd0a 100644 --- a/server/cli/maintenance.py +++ b/server/cli/maintenance.py @@ -256,7 +256,12 @@ async def _go() -> int: ) return 1 if rep.failed and rep.failed == rep.sessions else 0 finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/cli/sql_access.py b/server/cli/sql_access.py index 6a727c9e7b..c34dc37f25 100644 --- a/server/cli/sql_access.py +++ b/server/cli/sql_access.py @@ -82,7 +82,12 @@ async def _go() -> Any: await check_acceptance(orgs=orgs, org_free=org_free) return await work(ch) finally: - await disconnect(ch) + first = sys.exception() + try: + await disconnect(ch) + except BaseException: # a disconnect failure must not replace the command failure + if first is None: + raise return asyncio.run(_go()) diff --git a/server/core/retirement.py b/server/core/retirement.py index 025a437362..f3dd7fc865 100644 --- a/server/core/retirement.py +++ b/server/core/retirement.py @@ -31,6 +31,9 @@ def __init__(self) -> None: self.enrolled = False self.producers_stopped = False self.producers: set[asyncio.Task[Any]] = set() + # Native completion needs the first failed producer after its Task is discarded. + # Fresh gates, including fork reset, discard this one command's outcome. + self.producer_failure: BaseException | None = None def install(self, role: str) -> None: if self.closed or self.role not in ("", role): @@ -43,12 +46,19 @@ def close(self) -> None: def track[T](self, task: asyncio.Task[T]) -> asyncio.Task[T]: """A process-lifetime producer must actually end, not merely be canceled.""" self.producers.add(task) - task.add_done_callback(self.producers.discard) + task.add_done_callback(self._producer_done) if self.closed: task.cancel() raise ProcessStopping("Cannot start a producer during shutdown") return task + def _producer_done(self, task: asyncio.Task[Any]) -> None: + self.producers.discard(task) + if self.role == "command" and not task.cancelled(): + error = task.exception() + if self.producer_failure is None: + self.producer_failure = error + def stopped(self) -> None: """Composition calls this after stopping every role-specific producer.""" self.close() diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 193fcc9335..794b457649 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -264,15 +264,8 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> Every attempt, settle and pause ends by `budget_s` from now.""" from server.core import rmt_carry - deadline = monotonic() + budget_s try: - async with asyncio.timeout(budget_s): - while True: - if await retire_process(): - return True - with contextlib.suppress(Exception): - await rmt_carry.settle(ch) - await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + return await _retire_until(ch, monotonic() + budget_s) except TimeoutError: log.warning( "provider participant not retired at shutdown: %d carried write(s) unsettled " @@ -286,6 +279,19 @@ async def retire_at_shutdown(ch: Any, *, budget_s: float = SHUTDOWN_RETIRE_S) -> return False +async def _retire_until(ch: Any, deadline: float) -> bool: + """Retry the existing settlement checks under the caller's absolute deadline.""" + from server.core import rmt_carry + + async with asyncio.timeout_at(deadline): + while True: + if await retire_process(): + return True + with contextlib.suppress(Exception): + await rmt_carry.settle(ch) + await asyncio.sleep(min(1.0, max(0.0, deadline - monotonic()))) + + def participant_lifespan[T]( lifespan: Callable[[T], AbstractAsyncContextManager[None]], ) -> Callable[[T], AbstractAsyncContextManager[None]]: @@ -314,8 +320,12 @@ async def wrapped(app: T) -> AsyncIterator[None]: async def retire_participant(participant: UUID, *, ch: Any = None) -> str: - """`retired`, `absent` (no live row), or `live`: its heartbeat is fresh or a - query tagged with its id still runs.""" + """`retired`, `absent` (no row), or `live` (not proved stopped). + + Retirement needs the recorded identity proved absent in the same host, boot + and PID namespace, with tagged queries known zero. Unknown evidence blocks; + heartbeat expiry alone proves nothing. + """ row = next( (r for r in await participants(ch=ch) if str(r["participant_id"]) == str(participant)), None, @@ -342,9 +352,10 @@ async def retire_participant(participant: UUID, *, ch: Any = None) -> str: async def retire_dead_here(*, now: float | None = None) -> int: - """Retire the participants this host proves stopped (`proves_stopped`: a - restarted container or a respawned worker) with no query of theirs left - running. Every process's job, every `HERE_S`; how many it retired.""" + """Retire recorded process-start identities proved absent in this host, + boot and PID namespace, with tagged queries known zero. Unknown evidence + blocks. Run at most every `HERE_S`; return how many were retired. + """ global _last_here t = monotonic() if now is None else now if t - _last_here < HERE_S: diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 3b9f7d8a4f..3e16b03c48 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -9,9 +9,9 @@ from server.core import retirement, settings_overrides from server.core.background import drain_task_sets, flush_registered_buffers -from server.core.ch import set_query_tag +from server.core.ch import get_client, set_query_tag from server.core.model_work_gate import get_model_work_gate -from server.insights.app.provider_barrier import register_process, retire_process +from server.insights.app.provider_barrier import _retire_until, register_process from server.insights.app.provider_runtime import load_active_configuration from server.settings import settings @@ -47,49 +47,64 @@ async def _keep_overrides( return error +async def _drain_participant_until(end: float) -> None: + loop = asyncio.get_running_loop() + async with asyncio.timeout_at(end): + if not await drain_task_sets(timeout_s=max(0.0, end - loop.time())): + raise RuntimeError("Native command deferred writes did not drain") + if not await flush_registered_buffers(): + raise RuntimeError("Native command buffers did not flush") + if await _retire_until(get_client(), end) is not True: + raise RuntimeError("Native command retirement was not certified") + + async def _finish_participant() -> None: gate = retirement.get_gate() gate.close() from server.memory.app import embedder_calibration - first: BaseException | None = None + loop = asyncio.get_running_loop() + end = loop.time() + COMMAND_DRAIN_S + first = gate.producer_failure stopped = False try: embedder_calibration.stop_refresh_loop() stopped = True except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure - first = error + if first is None: + first = error producers = tuple(gate.producers) for producer in producers: producer.cancel() try: if producers: - _, pending = await asyncio.wait(producers, timeout=COMMAND_DRAIN_S) + _, pending = await asyncio.wait(producers, timeout=max(0.0, end - loop.time())) if pending: raise RuntimeError("Native command producers did not stop") except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure if first is None: first = error stopped = False - for producer in producers: - if producer.done() and not producer.cancelled(): + if first is None: + first = gate.producer_failure + if stopped: + gate.stopped() + owner = asyncio.current_task() + while True: + cancelling = owner.cancelling() if owner is not None else 0 try: - producer.result() - except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure + if loop.time() >= end: + raise TimeoutError("Native command cleanup deadline expired") + await _drain_participant_until(end) + except asyncio.CancelledError as error: if first is None: first = error - if stopped: - gate.stopped() - try: - if not await drain_task_sets(timeout_s=COMMAND_DRAIN_S): - raise RuntimeError("Native command deferred writes did not drain") - if not await flush_registered_buffers(): - raise RuntimeError("Native command buffers did not flush") - if await retire_process() is not True: - raise RuntimeError("Native command retirement was not certified") - except BaseException as error: # noqa: BLE001 — finish safe cleanup, then raise the first failure - if first is None: - first = error + if owner is not None and owner.cancelling() > cancelling: + continue # a new owner cancellation still allows cleanup within the same end + except BaseException as error: # noqa: BLE001 — report the first required cleanup failure + if first is None: + first = error + break if first is not None: raise first diff --git a/tests/unit/test_cell_trigger_gate_json_codec.py b/tests/unit/test_cell_trigger_gate_json_codec.py new file mode 100644 index 0000000000..62c9aefa46 --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_json_codec.py @@ -0,0 +1,201 @@ +"""The full stored-row proof accepts native JSON without weakening other fields.""" + +from __future__ import annotations + +import json +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from bench import cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import ( + args, + stored_bfloat16, +) +from tests.unit.test_bench_cell_trigger_gate import ( + no_external_io as no_external_io, +) + + +def native_json(monkeypatch, value): + from clickhouse_connect.datatypes import dynamic + from clickhouse_connect.datatypes.base import TypeDef + from clickhouse_connect.driver.context import BaseQueryContext + + # A fresh type leaves the installed registry and global format map untouched. + dtype = dynamic.JSON(TypeDef()) + context = BaseQueryContext() + assert dtype.read_format(context) == "native" + dtype.typed_paths = list(value) + dtype.typed_types = [ + SimpleNamespace(read_column_data=lambda *_args, v=v: [v]) for v in value.values() + ] + monkeypatch.setattr( + dynamic, + "SHARED_DATA_TYPE", + SimpleNamespace(read_column_data=lambda *_args: [{}]), + ) + state = dynamic.JSONState(2, [], [None] * len(value), [], None) + decoded = dtype._read_column_binary(object(), 1, context, state)[0] + assert type(decoded) is dict + assert decoded == value + return decoded + + +@pytest.mark.parametrize( + "value", + [ + {}, + {"n": 1}, + {"flag": True}, + {"nullable": None}, + {"nested": {"b": [1, False, None], "a": "text"}}, + {"when": "2026-01-02T03:04:05.123456+01:00"}, + {"text": '\u03bb\n"quoted"'}, + {"z": 0, "a": "1"}, + ], +) +def test_installed_native_json_matches_serialized_authority(monkeypatch, value): + decoded = native_json(monkeypatch, value) + gate._compare( + {"value_json": json.dumps(value, indent=2)}, + {"value_json": decoded}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize( + ("expected", "actual"), + [ + ({}, {"extra": 1}), + ({"n": 1}, {"n": True}), + ({"n": 1}, {"n": "1"}), + ({"v": None}, {}), + ({"v": [1, 2]}, {"v": [2, 1]}), + ({"v": {}}, {"v": []}), + ({"v": "2026-01-02T03:04:05.123456"}, {"v": "2026-01-02T03:04:05.123999"}), + ({"v": "2026-01-02T03:04:05+01:00"}, {"v": "2026-01-02T03:04:05+00:00"}), + ], +) +def test_native_json_rejects_changed_json_value(monkeypatch, expected, actual): + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare( + {"value_json": json.dumps(expected)}, + {"value_json": native_json(monkeypatch, actual)}, + ("value_json",), + "full memory projection", + ) + + +@pytest.mark.parametrize("side", ["expected", "actual"]) +def test_malformed_json_is_an_invalid_projection(side): + expected, actual = {"value_json": "{}"}, {"value_json": {}} + (expected if side == "expected" else actual)["value_json"] = "{broken" + with pytest.raises(gate.Invalid, match="full memory projection value_json"): + gate._compare(expected, actual, ("value_json",), "full memory projection") + + +def test_json_normalization_applies_only_to_value_json(): + with pytest.raises(gate.Invalid, match="full memory projection value"): + gate._compare({"value": "{}"}, {"value": {}}, ("value",), "full memory projection") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + ["", "value_json", "value", "version", "vector", "audit_hash"], + ids=["healthy", "value_json", "value", "version", "vector", "audit_hash"], +) +async def test_actual_verify_keeps_full_proof_with_native_json(monkeypatch, fault): + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + from server.memory.store.rows import MEMORY_PERSISTED_COLS + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + block.seqs = [("org", "ns", i + 1) for i in range(5)] + assert len(MEMORY_PERSISTED_COLS) == 61 + assert all(set(row) == set(MEMORY_PERSISTED_COLS) for row in block.rows) + records = [ + { + "head": json.dumps(pg_memory.encode_row(row)), + "revision": json.dumps(pg_memory.encode_row(row)), + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": i + 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + ] + projections = [ + {**row, "value_json": native_json(monkeypatch, json.loads(row["value_json"]))} + for row in block.rows + ] + documents = [dict(row) for row in block.documents] + for document in documents: + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + audits = [dict(row) for row in block.audits] + if fault == "value_json": + projections[0]["value_json"] = native_json(monkeypatch, {"wrong": True}) + elif fault == "value": + projections[0]["value"] = '"wrong"' + elif fault == "version": + projections[0]["version"] -= 1 + elif fault == "vector": + documents[0]["embedding"] = [0.0] * 16 + elif fault == "audit_hash": + audits[0]["row_hash"] = "bad" + conn = SimpleNamespace( + fetch=AsyncMock(return_value=records), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + phases = [] + compare = gate._compare + + def full_compare(expected, actual, fields, phase): + if phase == "full memory projection": + assert set(fields) == set(block.rows[0]) and len(fields) == 61 + phases.append(phase) + return compare(expected, actual, fields, phase) + + monkeypatch.setattr(gate, "_compare", full_compare) + + async def query(sql, params): + assert params["org"] == "org" and params["ns"] == "ns" + if "FROM loom.memory FINAL" in sql: + return projections + if "FROM loom.memory_embeddings FINAL" in sql: + return documents + if "FROM loom.audit_log" in sql: + return audits + raise AssertionError(sql) + + if fault: + reason = { + "value_json": "full memory projection value_json", + "value": "full memory projection value$", + "version": "full memory projection version", + "vector": "full search document vector", + "audit_hash": "exact stored audit hashes", + }[fault] + with pytest.raises(gate.Invalid, match=reason): + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + else: + await gate._verify(conn, SimpleNamespace(query_all=query), args(), block) + assert phases.count("full memory projection") == 5 + assert phases.count("full search document") == 5 + assert phases.count("full stored audit") == 5 + conn.fetchrow.assert_awaited_once() diff --git a/tests/unit/test_cli_native_completion.py b/tests/unit/test_cli_native_completion.py new file mode 100644 index 0000000000..6b27052577 --- /dev/null +++ b/tests/unit/test_cli_native_completion.py @@ -0,0 +1,252 @@ +"""Native completion waits for real retirement checks and observes tracked outcomes.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.cli import _ch +from server.core import retirement, rmt_carry +from server.insights.app import provider_barrier as barrier +from server.insights.app import provider_command as command +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_healthy_heartbeat_finishes_after_native_owner_requests_stop( + native, monkeypatch, first_class +): + requested = [] + attempts = [] + heartbeat = SimpleNamespace(running=True) + + def stop(): + requested.append(True) + + async def settle(client): + assert client is native.client + assert requested and heartbeat.running + attempts.append(True) + heartbeat.running = False + + heartbeat.signal_stop = stop + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + first = first_class("work") if first_class else None + + async def work(_): + if first is not None: + raise first + return 7 + + if first is None: + assert _ch.run_connected(work, orgs=("o",)) == 7 + else: + with pytest.raises(first_class) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert attempts == [True] + assert not heartbeat.running + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_permanent_heartbeat_respects_same_absolute_native_allowance( + native, monkeypatch, first_class +): + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr( + barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=True) + ) + monkeypatch.setattr(rmt_carry, "settle", AsyncMock()) + first = first_class("work") if first_class else None + elapsed = [] + + async def work(_): + loop = asyncio.get_running_loop() + elapsed.append((loop, loop.time())) + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or TimeoutError) as caught: + _ch.run_connected(work, orgs=("o",)) + if first is not None: + assert caught.value is first + loop, start = elapsed[0] + assert loop.time() - start < 0.5 + assert "certificate" not in native.events + assert native.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, ValueError, asyncio.CancelledError]) +def test_repeated_owner_cancellation_keeps_first_and_allows_finite_heartbeat_completion( + native, monkeypatch, first_class +): + first = first_class("work") if first_class else None + heartbeat = SimpleNamespace(running=True, signal_stop=lambda: None) + attempts = [] + cancellations = [] + + async def settle(client): + assert client is native.client + owner = asyncio.current_task() + attempts.append(True) + if len(attempts) <= 2: + owner.cancel("first signal" if len(attempts) == 1 else "second signal") + else: + heartbeat.running = False + try: + await asyncio.sleep(0) + except asyncio.CancelledError as error: + cancellations.append(error) + raise + + monkeypatch.setattr(barrier, "_HEARTBEAT", heartbeat) + monkeypatch.setattr(rmt_carry, "settle", settle) + + async def work(_): + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else cancellations[0]) + assert len(attempts) == 3 and len(cancellations) == 2 + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", [None, *FIRST]) +def test_real_track_completed_failure_is_observed_before_native_cleanup(native, first_class): + producer_error = OSError("producer") + first = first_class("work") if first_class else None + tasks = [] + + async def work(_): + async def producer(): + raise producer_error + + task = native.gate.track(asyncio.create_task(producer())) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + if first is not None: + raise first + return 7 + + with pytest.raises(first_class or OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is (first if first is not None else producer_error) + assert native.events[-3:] == ["certificate", "pg", "ch"] + assert all(task.done() and not task._log_traceback for task in tasks) + + +@pytest.mark.asyncio +async def test_native_track_retains_only_first_failure_and_no_completed_task_handles(): + gate = retirement.RetirementGate() + gate.install("command") + first = OSError("first producer") + tasks = [] + + async def producer(error=None): + if error is not None: + raise error + + for error in [None] * 100 + [first, ValueError("later")] + [None] * 100: + task = gate.track(asyncio.create_task(producer(error))) + tasks.append(task) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producer_failure is first + assert gate.producers == set() + assert all(not task._log_traceback for task in tasks) + assert retirement.RetirementGate().producer_failure is None + + +@pytest.mark.asyncio +async def test_intentionally_cancelled_tracked_native_producer_is_not_failed(): + gate = retirement.RetirementGate() + gate.install("command") + task = gate.track(asyncio.create_task(asyncio.Event().wait())) + await asyncio.sleep(0) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() and gate.producers == set() + assert gate.producer_failure is None + + +@pytest.mark.parametrize("role", ["server", "worker"]) +@pytest.mark.asyncio +async def test_other_role_tracking_keeps_existing_outcome_ownership(role): + gate = retirement.RetirementGate() + gate.install(role) + first = OSError("role-owned producer") + + async def producer(): + raise first + + task = gate.track(asyncio.create_task(producer())) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert gate.producers == set() and gate.producer_failure is None + assert task._log_traceback + assert task.exception() is first + + +def test_actual_calibration_consumer_failure_survives_finished_track_callback(native, monkeypatch): + from server.memory.app import embedder_calibration + + first = OSError("calibration producer") + tasks = [] + + async def sleep(_): + raise first + + monkeypatch.setattr(embedder_calibration, "_refresh_task", None) + monkeypatch.setattr( + embedder_calibration, + "asyncio", + SimpleNamespace(sleep=sleep, get_running_loop=asyncio.get_running_loop), + ) + + async def configuration(): + tasks.append(embedder_calibration.start_refresh_loop()) + + monkeypatch.setattr(command, "load_active_configuration", configuration) + + async def work(_): + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not native.gate.producers + return 7 + + with pytest.raises(OSError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert tasks[0].done() and not tasks[0]._log_traceback + assert native.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("phase", ["flush", "certificate"]) +def test_dependency_cancellation_is_not_retried_as_owner_cancellation(native, monkeypatch, phase): + first = asyncio.CancelledError("cleanup dependency cancelled") + required = AsyncMock( + side_effect=[first, RuntimeError("cleanup retried without owner cancellation")] + ) + target = "flush_registered_buffers" if phase == "flush" else "_retire_until" + monkeypatch.setattr(command, target, required) + + async def work(_): + return 7 + + with pytest.raises(asyncio.CancelledError) as caught: + _ch.run_connected(work, orgs=("o",)) + assert caught.value is first + assert required.await_count == 1 + assert native.events[-2:] == ["pg", "ch"] + assert not native.client.connected + assert native.gate.closed and native.gate.active == 0 diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py index 35f71b60fb..c77d29ba83 100644 --- a/tests/unit/test_cli_native_first_error.py +++ b/tests/unit/test_cli_native_first_error.py @@ -67,6 +67,7 @@ async def certificate(proof): client.connect, client.disconnect = connect, close_ch monkeypatch.setattr(ch, "get_client", lambda: client) + monkeypatch.setattr(command, "get_client", lambda: client, raising=False) monkeypatch.setattr(pg, "close_pool", close_pg) monkeypatch.setattr(pg_profile, "check_configured", verify) monkeypatch.setattr(_ch, "check_orgs", acceptance) @@ -218,14 +219,14 @@ def test_unacknowledged_cleanup_refuses_without_early_certification( calls = [] first = first_class("work") if first_class else None - async def unacknowledged(**_): + async def unacknowledged(*_, **__): calls.append(phase) return False target = { "drain": "drain_task_sets", "flush": "flush_registered_buffers", - "retire": "retire_process", + "retire": "_retire_until", } monkeypatch.setattr(command, target[phase], unacknowledged) @@ -381,12 +382,13 @@ async def work(_): async def producer(error): raise error - # Hold completed producer handles until the composition retrieves both errors. + # Completion callbacks run before native shutdown snapshots tracked producers. for error in [first, first]: task = asyncio.create_task(producer(error)) - native.gate.producers.add(task) + native.gate.track(task) producers.append(task) await asyncio.sleep(0) + await asyncio.sleep(0) return 7 with pytest.raises(OSError) as caught: @@ -397,6 +399,7 @@ async def producer(error): def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): + monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) stopped = [] monkeypatch.setattr( barrier, @@ -407,9 +410,9 @@ def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, async def work(_): return 7 - with pytest.raises(RuntimeError, match="retirement was not certified"): + with pytest.raises(TimeoutError): _ch.run_connected(work, orgs=("o",)) - assert stopped == [True] + assert stopped and all(stopped) assert "certificate" not in native.events assert native.gate.enrolled and native.gate.quiescent assert native.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_cli_native_wrappers.py b/tests/unit/test_cli_native_wrappers.py new file mode 100644 index 0000000000..72283e6512 --- /dev/null +++ b/tests/unit/test_cli_native_wrappers.py @@ -0,0 +1,316 @@ +"""Actual native entry points preserve first failures across their outer close.""" + +import argparse +import asyncio +import sqlite3 +from functools import partial +from types import SimpleNamespace + +import pytest + +from server.cli import ( + _ch, + analytics_export, + code_run, + import_brain, + import_clickgap, + import_recall, + maintenance, + sql_access, +) +from server.insights.app import analytics_export as export_app +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.fixture +def wrappers(native, monkeypatch): + from server.code.app.index import repo_registry + from server.memory.app import reextract + + monkeypatch.setattr(repo_registry, "refresh_db_repos", _nothing) + monkeypatch.setattr(analytics_export, "_refusal", lambda *_: "") + monkeypatch.setattr(export_app, "check_credentials", lambda **_: None) + monkeypatch.setattr(export_app, "name_hash_key", lambda _: b"inert") + monkeypatch.setattr(export_app, "s3_put", lambda *_, **__: None) + monkeypatch.setattr(maintenance, "resolve_token", lambda _: ("", "none")) + monkeypatch.setattr(reextract, "replay", _report) + monkeypatch.setattr(reextract, "render_report", lambda *_, **__: "report") + return native + + +async def _nothing(*_, **__): + return None + + +async def _report(*_, **__): + return SimpleNamespace(failed=0, sessions=1) + + +def _invoke(family, work, monkeypatch): + if family == "sql": + return sql_access._connected(work, orgs=("o",)) + if family == "export": + monkeypatch.setattr(export_app, "export_day", work) + return analytics_export._cmd_export( + argparse.Namespace( + day=None, + role_arn="", + region="", + bucket="scratch", + expected_bucket_owner="", + prefix="", + ) + ) + if family == "reextract": + from server.memory.app import reextract + + monkeypatch.setattr(reextract, "load_sessions", work) + return maintenance._cmd_reextract( + argparse.Namespace( + namespace="n", + into="other", + token="", + dry_run=True, + org="o", + limit_sessions=None, + base_url="", + dates=False, + concurrency=1, + ) + ) + if family == "code": + return asyncio.run(code_run._then_disconnect(work())) + raise AssertionError(family) + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("second_close", [False, True]) +def test_actual_wrappers_preserve_work_failure( + wrappers, monkeypatch, family, first_class, second_close +): + first = first_class("work") + if second_close: + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def work(*_, **__): + assert wrappers.gate.enrolled and wrappers.gate.active > 0 + raise first + + with pytest.raises(first_class) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert not wrappers.client.connected + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["sql", "export", "reextract", "code"]) +@pytest.mark.parametrize("failed_close", [False, True]) +def test_successful_actual_wrappers_still_require_cleanup( + wrappers, monkeypatch, family, failed_close +): + first = OSError("pg close") + if failed_close: + wrappers.errors.update(pg=first, ch=RuntimeError("ch close")) + + async def work(*_, **__): + return [] if family == "export" else ([{}] if family == "reextract" else 7) + + if failed_close: + with pytest.raises(OSError) as caught: + _invoke(family, work, monkeypatch) + assert caught.value is first + else: + assert _invoke(family, work, monkeypatch) == (0 if family in ("export", "reextract") else 7) + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("reopen_failure", [False, True]) +def test_code_reconnect_cannot_replace_subcommand_failure( + wrappers, monkeypatch, first_class, reopen_failure +): + first = first_class("subcommand") + events = [] + + async def reconnect(*_, **__): + events.append("reconnect") + if reopen_failure: + raise OSError("reconnect") + + async def work(): + raise first + + monkeypatch.setattr(code_run, "connect_declared", reconnect) + with pytest.raises(first_class) as caught: + asyncio.run(code_run._then_disconnect(work())) + assert caught.value is first + assert events == ["reconnect"] + assert wrappers.events[-3:] == ["certificate", "pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +@pytest.mark.parametrize("phase", ["entry", "participant-close"]) +@pytest.mark.asyncio +async def test_actual_recall_writer_entry_and_closer_preserve_first( + wrappers, monkeypatch, first_class, phase +): + first = first_class("first") + if phase == "entry": + wrappers.errors["acceptance"] = first + else: + wrappers.errors["certificate"] = first + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + args = argparse.Namespace(base_url=None, org="o") + with pytest.raises(first_class) as caught: + _, closer = await import_recall._make_writer(args, lambda: {}, timeout_s=1) + await closer() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + assert wrappers.gate.active == wrappers.model_gate.active == 0 + + +@pytest.mark.parametrize("family", ["recall", "brain"]) +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +def test_actual_importer_outer_closer_preserves_unhandled_work_failure( + wrappers, monkeypatch, tmp_path, family, first_class +): + first = first_class("write cancelled or fatal") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def handler(_): + raise first + + if family == "brain": + (tmp_path / "MEMORY.md").write_text("## A\ncontent\n") + monkeypatch.setattr(import_brain, "_brain_inproc_ops", lambda: {"profile.set": handler}) + args = argparse.Namespace( + dir=str(tmp_path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_brain._cmd_import_brain + else: + path = tmp_path / "source.db" + with sqlite3.connect(path): + pass + report = { + "tables": {}, + "memories_imported": 1, + "messages_imported": 0, + "threads": 0, + "skipped_rows": 0, + } + monkeypatch.setattr( + import_recall, + "_recall_plan_writes", + lambda *_, **__: ([{"op": "memory.set", "payload": {}}], report), + ) + monkeypatch.setattr(import_recall, "_recall_inproc_ops", lambda: {"memory.set": handler}) + args = argparse.Namespace( + sqlite=str(path), dry_run=False, org="o", namespace="n", agent="a", base_url=None + ) + call = import_recall._cmd_import_recall + with pytest.raises(first_class) as caught: + call(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.parametrize("first_class", FIRST) +def test_actual_clickgap_outer_disconnect_preserves_admission_failure( + wrappers, monkeypatch, tmp_path, first_class +): + from server import embedding + + first = first_class("admission") + wrappers.errors.update(acceptance=first, pg=OSError("pg close"), ch=RuntimeError("ch close")) + path = tmp_path / "source.db" + with sqlite3.connect(path) as source: + source.execute( + "CREATE TABLE lessons (id TEXT, text TEXT, source TEXT, created_at TEXT, kind TEXT, expires_at TEXT, evidence_count INT, recall_count INT, last_recalled_at TEXT, reasoning TEXT)" + ) + monkeypatch.setattr(import_clickgap, "_clickgap_count_source", lambda *_, **__: {}) + monkeypatch.setattr(import_clickgap, "_clickgap_preload", lambda *_, **__: ({}, [], {}, {})) + monkeypatch.setattr(embedding, "get_embedder", lambda: SimpleNamespace(model_name="inert")) + args = argparse.Namespace( + sqlite=str(path), + org="o", + namespace="n", + agent="a", + dry_run=False, + batch_size=1, + seed_recall_events=False, + ) + with pytest.raises(first_class) as caught: + import_clickgap._cmd_import_clickgap(args) + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + + +@pytest.mark.doctor_tier_read +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.parametrize("diagnostic", ["cell", "strays", "registry", "tiers"]) +def test_actual_diagnostic_closes_preserve_unhandled_native_failure( + wrappers, monkeypatch, first_class, diagnostic +): + from server.cli import doctor, doctor_cell + from server.code.app.index import repo_registry, tier_health + from server.core import settings_overrides + from server.settings import settings + + first = first_class("diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + + async def fail(*_, **__): + raise first + + if diagnostic == "cell": + monkeypatch.setattr(doctor_cell, "_checks", fail) + call = doctor_cell.cell_checks + elif diagnostic == "strays": + monkeypatch.setattr(doctor_cell, "_deployment_check", _nothing) + monkeypatch.setattr(doctor_cell, "_stray_reads", fail) + call = partial(doctor_cell.stray_org_checks, 1) + else: + monkeypatch.setattr(settings, "ch_host", "inert") + if diagnostic == "registry": + monkeypatch.setattr(repo_registry, "registry_read", lambda: False) + monkeypatch.setattr(repo_registry, "refresh_db_repos", fail) + call = doctor._read_code_registry + else: + monkeypatch.setattr(settings, "code_intel_enabled", True) + monkeypatch.setattr(doctor, "_read_code_registry", lambda: None) + monkeypatch.setattr(repo_registry, "registry_read", lambda: True) + monkeypatch.setattr(settings_overrides, "refresh", _nothing) + monkeypatch.setattr(tier_health, "refresh_failing", fail) + doctor._doctor_tier_failures.cache_clear() + call = doctor._doctor_tier_failures + with pytest.raises(first_class) as caught: + call() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] + doctor._doctor_tier_failures.cache_clear() + + +@pytest.mark.parametrize("first_class", [asyncio.CancelledError, SystemExit, KeyboardInterrupt]) +@pytest.mark.asyncio +async def test_actual_setup_store_diagnostic_preserves_unhandled_failure( + wrappers, monkeypatch, first_class +): + from server.core import pg, pg_profile + from server.settings import settings + + first = first_class("setup diagnostic") + wrappers.errors.update(pg=OSError("pg close"), ch=RuntimeError("ch close")) + monkeypatch.setattr(settings, "cell", "c") + monkeypatch.setattr(pg, "enabled", lambda: True) + + async def refusal(_): + raise first + + monkeypatch.setattr(pg_profile, "recorded_authority", refusal) + with pytest.raises(first_class) as caught: + await _ch.check_setup_stores() + assert caught.value is first + assert wrappers.events[-2:] == ["pg", "ch"] diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 2a1d22da2d..9a70d03b37 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -16,7 +16,7 @@ def setup(monkeypatch): gate = ModelWorkGate() monkeypatch.setattr(command, "get_model_work_gate", lambda: gate) monkeypatch.setattr(command, "register_process", AsyncMock()) - monkeypatch.setattr(command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(command, "_retire_until", AsyncMock(return_value=True)) monkeypatch.setattr(command, "load_active_configuration", AsyncMock()) monkeypatch.setattr(command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(command, "flush_registered_buffers", AsyncMock(return_value=True)) @@ -29,9 +29,9 @@ async def test_command_enrolls_loads_settings_then_admits_and_retires(setup): assert setup.active == 1 command.register_process.assert_awaited_once() command.load_active_configuration.assert_awaited_once() - command.retire_process.assert_not_awaited() + command._retire_until.assert_not_awaited() assert setup.active == 0 - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio @@ -40,7 +40,7 @@ async def test_pending_transition_never_runs_command(setup): with pytest.raises(ModelChangeInProgress): async with command.model_command(): pytest.fail("closed admission must reject a newborn command") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() @pytest.mark.asyncio @@ -49,7 +49,7 @@ async def test_undrained_descendants_keep_participant_registered(setup, monkeypa with pytest.raises(RuntimeError, match="deferred writes did not drain"): async with command.model_command(): pass - command.retire_process.assert_not_awaited() + command._retire_until.assert_not_awaited() command.flush_registered_buffers.assert_not_awaited() @@ -58,4 +58,4 @@ async def test_business_failure_does_not_skip_drain(setup): with pytest.raises(ValueError, match="business failure"): async with command.model_command(): raise ValueError("business failure") - command.retire_process.assert_awaited_once() + command._retire_until.assert_awaited_once() From b253487b6875bdc89cd4fa12057e2a5b8ecbdca1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 16:11:59 +0000 Subject: [PATCH 101/181] WIP: preserve reviewed attach lifecycle corrections Signed-off-by: zlareb1 --- docs/operations/upgrades.md | 14 ++-- server/cli/_ch.py | 6 +- server/cli/cell_attach_beats.py | 2 +- server/cli/cell_control.py | 2 +- server/core/schema/pg_migrate.py | 2 +- server/insights/app/provider_command.py | 4 +- server/router/directory.py | 2 +- tests/integration/test_cell_retirement.py | 80 +++++++++++++++++++++-- tests/unit/test_cell_retirement.py | 2 +- 9 files changed, 93 insertions(+), 21 deletions(-) diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 50dd5fc46a..02e45f6c80 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -306,9 +306,10 @@ checkout. refused as a permission error (SQLSTATE 42501) rather than `LT001`. 16. Before a first cell's attach, every CLI and CI image runs a build with that state read, - and every CLI that writes for an org has `LOOM_CELL` set. No CLI sends a heartbeat, so - attach cannot check either, and a CLI of an older build is not refused on an `enforced` - cell ([cells](../architecture/cells.md#failure-modes)). + and every CLI that writes for an org has `LOOM_CELL` set. A CLI publishes no instance + or cell heartbeat for attach to read, so attach cannot check either, and a CLI of an + older build is not refused on an `enforced` cell + ([cells](../architecture/cells.md#failure-modes)). 17. Moving to a build whose first-cell processes read their cell's state from the router on a timer: every router replica (`loom router serve`) runs this build before any cell's @@ -322,9 +323,10 @@ checkout. 18. Before a first cell's attach, once `LOOM_CELL` is set on every process and every CLI and CI configuration of the cell (step 16), rotate the ClickHouse credential they write - with: a new user or password for the cell's servers and CLIs, the old one dropped. No - CLI sends a heartbeat, so attach cannot see a job still configured with `LOOM_CELL` - unset; the rotation cuts off any the rollout missed before it can write an org's rows + with: a new user or password for the cell's servers and CLIs, the old one dropped. A + CLI publishes no instance or cell heartbeat for attach to read, so attach cannot see a + job still configured with `LOOM_CELL` unset; the rotation cuts off any the rollout + missed before it can write an org's rows past attach's last scan. Name the new user with `--ch-allow-user` ([attaching the first cell](./cell-attach.md)). diff --git a/server/cli/_ch.py b/server/cli/_ch.py index bca6dadbaa..65a4385e82 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -344,7 +344,7 @@ async def connect_verified(ch: CHClient) -> None: await ch.connect() try: await check_configured(ch) - except BaseException: + except BaseException: # Keep the pairing failure while closing its clients. with suppress(BaseException): await disconnect(ch) raise @@ -370,7 +370,7 @@ async def disconnect(ch: CHClient) -> None: pg.expect_deployment(None) try: await pg.close_pool() - except BaseException: + except BaseException: # Keep the pool close failure while closing ClickHouse. with suppress(BaseException): await ch.disconnect() raise @@ -402,7 +402,7 @@ async def _go() -> T: async with model_command(): await check_acceptance(orgs=named, org_free=org_free) result = await run(ch) - except BaseException: + except BaseException: # Keep the command failure while disconnecting its clients. with suppress(BaseException): await disconnect(ch) raise diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index c2a8ed2428..f49819a1e0 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -293,7 +293,7 @@ async def read_once(self) -> list[Beat]: try: async with asyncio.timeout(_READ_TIMEOUT_S): beats = await read_beats(self.ch, since=self.since) - except Exception: + except Exception: # A failed read still checks whether the heartbeat watch has a gap. self._check_gap() raise self._check_gap() diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 730c36bd42..bf412e2d08 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -146,7 +146,7 @@ async def _open(self, what: str) -> None: try: await pg._check_deployment(self.conn) await pg.init_connection(self.conn) - except Exception as error: + except Exception as error: # Classify unavailable openings; keep other failures. pg._raise_if_unavailable(error) raise refused = await apg.direct_refusals(self.conn, what) diff --git a/server/core/schema/pg_migrate.py b/server/core/schema/pg_migrate.py index a62b467c0c..4a0fbdf6bf 100644 --- a/server/core/schema/pg_migrate.py +++ b/server/core/schema/pg_migrate.py @@ -119,7 +119,7 @@ async def migrate( result = await asyncio.shield(task) if isinstance(result, (SystemExit, KeyboardInterrupt)): raise result - except BaseException as error: + except BaseException as error: # Keep the migration failure through owned cleanup. end = time.monotonic() + pg._RELEASE_TIMEOUT_S await _failed_migration(cm, conn, task, error, end) raise diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index 3e16b03c48..dd16904ddb 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -117,7 +117,7 @@ async def participant_command() -> AsyncIterator[None]: try: async with retirement.get_gate().admit(): yield - except BaseException: + except BaseException: # Keep the body failure while attempting required retirement cleanup. with contextlib.suppress(BaseException): await _finish_participant() raise @@ -154,6 +154,6 @@ async def model_command() -> AsyncIterator[None]: else: if fatal is not None: raise fatal - except BaseException: + except BaseException: # Keep the body's first failure through keeper cleanup. if first is None: raise diff --git a/server/router/directory.py b/server/router/directory.py index bdf9f4bdd6..3e438cf6d9 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1014,7 +1014,7 @@ async def release(self) -> None: self.exit_task = self.start(lambda: self.context.__aexit__(None, None, None)) try: await self.wait(self.exit_task, self.end) - except BaseException: + except BaseException: # An interrupted pool exit must not continue without its owner. self.exit_task.cancel() raise diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py index 0652b4403f..a769a0728e 100644 --- a/tests/integration/test_cell_retirement.py +++ b/tests/integration/test_cell_retirement.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import contextlib +import json import os import sys from uuid import UUID, uuid4 @@ -11,26 +13,74 @@ from server.core import provider_process, retirement from server.core.ch import CHClient, query_tag +from server.core.sql_literal import sql_string from server.insights.store import provider_participants as store +def _write_scratch_owner(tmp_path, owner, **changes): + owner.update(changes) + owner["cleanup_complete"] = owner["drop_confirmed"] and owner["disconnect_confirmed"] + path = tmp_path / "scratch-owner.json" + staging = tmp_path / "scratch-owner.next" + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(owner, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(staging, path) + descriptor = os.open(tmp_path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _scratch_owner(tmp_path, request, database, table): + owner = { + "schema": "loom.native-scratch.v1", + "case_nodeid": request.node.nodeid, + "database": database, + "table": table, + "ownership_marker": "loom-native-retirement-" + uuid4().hex, + "create_intended": True, + "create_confirmed": False, + "table_created": False, + "drop_confirmed": False, + "disconnect_confirmed": False, + } + _write_scratch_owner(tmp_path, owner) + return owner + + @pytest.mark.asyncio async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_history( monkeypatch, + tmp_path, + request, ): if not os.environ.get("LOOM_CH_HOST"): pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") database = "retirement_" + uuid4().hex table = database + ".participants" + owner = _scratch_owner(tmp_path, request, database, table) client = CHClient() - await client.connect() + created = False with query_tag("test.cell_retirement"): - await client.execute("CREATE DATABASE {database:Identifier}", {"database": database}) try: + await client.connect() + await client.execute( + "CREATE DATABASE {database:Identifier} COMMENT " + + sql_string(owner["ownership_marker"]), + {"database": database}, + ) + created = True + _write_scratch_owner(tmp_path, owner, create_confirmed=True) await client.execute( "CREATE TABLE {database:Identifier}.participants AS loom._provider_participants", {"database": database}, ) + _write_scratch_owner(tmp_path, owner, table_created=True) old_table = store.TABLE for name in ( "TABLE", @@ -78,7 +128,8 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ ], ) rows = await client.query_all( - f"SELECT participant_id, transition_id, state, version, stale_after_s, host_proof FROM {table} FINAL" + "SELECT participant_id, transition_id, state, version, stale_after_s, host_proof " + f"FROM {table} FINAL" ) final = next(row for row in rows if str(row["participant_id"]) == str(stopped)) assert str(final["transition_id"]) == str(UUID(int=0)) @@ -99,5 +150,24 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ (str(legacy), "retired") ] finally: - await client.execute("DROP DATABASE {database:Identifier} SYNC", {"database": database}) - await client.disconnect() + # Keep the test's failure if required scratch cleanup also fails. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + try: + if created: + await client.execute( + "DROP DATABASE {database:Identifier} SYNC", {"database": database} + ) + _write_scratch_owner(tmp_path, owner, drop_confirmed=True) + finally: + # Disconnect must not replace a failed drop or receipt write. + with ( + contextlib.suppress(BaseException) + if sys.exception() is not None + else contextlib.nullcontext() + ): + await client.disconnect() + _write_scratch_owner(tmp_path, owner, disconnect_confirmed=True) diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index a5a3c8dcce..178ba2bd1f 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -651,7 +651,7 @@ async def work(ch): monkeypatch.setattr(provider_command, "register_process", register) monkeypatch.setattr(provider_command, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(provider_command, "flush_registered_buffers", AsyncMock(return_value=True)) - monkeypatch.setattr(provider_command, "retire_process", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "_retire_until", AsyncMock(return_value=True)) monkeypatch.setattr(_ch, "check_orgs", acceptance) assert sql_access._connected(work, orgs=("example",)) == 7 assert retirement.get_gate().closed and retirement.get_gate().active == 0 From d182c26bd532d69dacc159ff022e40be706b8dc2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 16:59:47 +0000 Subject: [PATCH 102/181] WIP: preserve attach registration and error regressions Signed-off-by: zlareb1 --- tests/unit/test_bench_cell_trigger_gate.py | 2 +- tests/unit/test_cell_attach.py | 7 +- .../unit/test_cell_attach_serving_identity.py | 134 ++++++++++++++++++ .../test_cell_trigger_gate_error_envelope.py | 86 +++++++++++ tests/unit/test_platform_admin_rollout.py | 46 +++++- 5 files changed, 272 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_cell_attach_serving_identity.py create mode 100644 tests/unit/test_cell_trigger_gate_error_envelope.py diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 9d9ce8f180..7543067fd6 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -1143,7 +1143,7 @@ async def test_review_failed_request_keeps_raw_status_elapsed_and_classification ): async def send(self, request, **kwargs): return httpx.Response( - status, json={"error": {"code": code, "message": "secret-token"}}, request=request + status, json={"detail": {"code": code, "message": "secret-token"}}, request=request ) monkeypatch.setattr(httpx.AsyncClient, "send", send) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 5188641547..819f03d90f 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -964,10 +964,14 @@ async def test_a_serving_row_is_done_only_once_the_cell_is_enforced( from server.core import pg state = ["enforced"] + conn = FakeConn([("SELECT deployment_id FROM loom.authority", "dep-1")]) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "") + monkeypatch.setattr(settings, "document_s3_prefix", "") @asynccontextmanager async def dedicated(**_kw: Any) -> Any: - yield None + yield conn async def read_mode(_conn: Any) -> tuple[str, Any]: return state[0], (True, "first") @@ -976,6 +980,7 @@ async def read_mode(_conn: Any) -> tuple[str, Any]: monkeypatch.setattr(apg, "read_mode", read_mode) run = cell_attach.Run(None, _args(), None, lambda _t: None) # type: ignore[arg-type] run.owner = _owner(FakeConn()) + run.row = {"deployment_id": "dep-1", "cell": "first", "bucket": "", "bucket_prefix": ""} assert await run.done() state[0] = "recording" with pytest.raises(Refused, match="attach lowers and repairs neither"): diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py new file mode 100644 index 0000000000..469ed16ea3 --- /dev/null +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -0,0 +1,134 @@ +"""A serving shortcut validates the registration without claiming or writing.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_attach, cell_control +from server.cli.cell_ch import Refused +from server.core import pg +from server.settings import settings +from tests.unit.test_cell_attach import FakeConn, _args + + +@pytest.fixture +def serving_run(monkeypatch): + def make( + *, + deployment="registered", + configured_cell="first", + bucket="bucket", + prefix="docs/", + first=True, + mode="enforced", + ): + row = { + "cell": "first", + "is_first": first, + "state": "serving", + "host": "first.example", + "deployment_id": "registered", + "bucket": "bucket", + "bucket_prefix": "docs/", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": str(UUID(int=1)), + } + monkeypatch.setattr(settings, "cell", configured_cell) + monkeypatch.setattr(settings, "document_s3_bucket", bucket) + monkeypatch.setattr(settings, "document_s3_prefix", prefix) + conn = FakeConn( + [ + ("SELECT deployment_id FROM loom.authority", deployment), + ( + "SELECT (SELECT is_first FROM loom.cell_first)", + { + "is_first": first, + "cell": "first", + "mode": mode, + "has_own": True, + "has_tenancy": True, + }, + ), + ( + "SELECT is_first, cell FROM loom.cell_first", + {"is_first": first, "cell": "first"}, + ), + ] + ) + sessions = [] + + @asynccontextmanager + async def dedicated(**kwargs): + sessions.append("opened") + try: + yield conn + finally: + sessions.append("closed") + + monkeypatch.setattr(pg, "dedicated", dedicated) + owner = AsyncMock(side_effect=AssertionError("serving must not start an owner")) + claim = AsyncMock(side_effect=AssertionError("serving must not claim")) + mutate = AsyncMock(side_effect=AssertionError("serving must not mutate")) + monkeypatch.setattr(cell_control, "run", owner) + monkeypatch.setattr(cell_control, "claim", claim) + monkeypatch.setattr(cell_control, "mutate", mutate) + router = SimpleNamespace(call=AsyncMock(return_value=row)) + run = cell_attach.Run(None, _args(), router, lambda text: None) + return run, conn, router, sessions, (owner, claim, mutate) + + return make + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides", + [ + pytest.param({"deployment": "another-deployment"}, id="deployment"), + pytest.param({"deployment": None}, id="missing-authority"), + pytest.param({"configured_cell": "other"}, id="configured-cell"), + pytest.param({"bucket": "another-bucket"}, id="bucket"), + pytest.param({"prefix": "another-prefix/"}, id="bucket-prefix"), + ], +) +async def test_serving_run_refuses_another_registration_without_writes(serving_run, overrides): + run, conn, router, sessions, mutations = serving_run(**overrides) + with pytest.raises(Refused, match="do not match the directory registration"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_args_list[0].args == ("",) + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first", [True, False], ids=["first", "later"]) +async def test_matching_serving_registration_is_done_without_writes(serving_run, first): + run, conn, router, sessions, mutations = serving_run(first=first) + assert await run.run() == 0 + assert run.report == {"cell": "first", "state": "serving", "done": True} + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) + + +@pytest.mark.asyncio +async def test_matching_serving_registration_still_refuses_recording(serving_run): + run, conn, router, sessions, mutations = serving_run(mode="recording") + with pytest.raises(Refused, match="attach lowers and repairs neither"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert router.call.await_count == 1 + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) diff --git a/tests/unit/test_cell_trigger_gate_error_envelope.py b/tests/unit/test_cell_trigger_gate_error_envelope.py new file mode 100644 index 0000000000..2bdf47074e --- /dev/null +++ b/tests/unit/test_cell_trigger_gate_error_envelope.py @@ -0,0 +1,86 @@ +"""Failed benchmark requests classify real wire errors without echoing payloads.""" + +import json + +import httpx +import pytest +from fastapi import FastAPI + +from bench import cell_trigger_gate as gate +from server.core.errors import register_exception_handler +from server.core.pg_lease import PublicationPending +from tests.unit.test_bench_cell_trigger_gate import args + + +@pytest.fixture(autouse=True) +def no_database(monkeypatch): + async def forbidden(*args, **kwargs): + raise AssertionError("error classification must not open a database") + + monkeypatch.setattr(gate.asyncpg, "connect", forbidden) + monkeypatch.setattr(gate.clickhouse_connect, "get_async_client", forbidden) + + +@pytest.mark.asyncio +async def test_publication_pending_from_the_server_handler_retains_its_code(monkeypatch): + app = FastAPI() + register_exception_handler(app) + + @app.post("/v1/memory.set") + async def pending(): + error = PublicationPending() + error.message = "secret-token" + raise error + + transport = httpx.ASGITransport(app=app) + envelopes = [] + + async def send(self, request, **kwargs): + response = await transport.handle_async_request(request) + await response.aread() + envelopes.append(response.json()) + return response + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(envelopes) == 1 and "error" not in envelopes[0] + assert envelopes[0]["detail"]["code"] == "PUBLICATION_PENDING" + assert envelopes[0]["detail"]["committed"] is True + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == "PUBLICATION_PENDING" + assert events[0][1]["measured"] is False + assert "secret-token" not in json.dumps(events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "candidate,expected", + [ + pytest.param("BAD_CONFIG", "BAD_CONFIG", id="valid-code"), + pytest.param("A" * 64, "A" * 64, id="maximum-length"), + pytest.param("secret-token", "", id="unsafe-text"), + pytest.param("A" * 65, "", id="overlong"), + pytest.param(42, "", id="number"), + pytest.param(None, "", id="null"), + pytest.param([], "", id="list"), + ], +) +async def test_detail_code_uses_the_existing_allowlist(monkeypatch, candidate, expected): + async def send(self, request, **kwargs): + return httpx.Response( + 503, + request=request, + json={"detail": {"code": candidate, "message": "secret-token"}}, + ) + + monkeypatch.setattr(httpx.AsyncClient, "send", send) + events = [] + with pytest.raises(gate.Invalid, match="memory.set HTTP status"): + await gate._memory_set(args(), emit=lambda kind, **row: events.append((kind, row))) + assert len(events) == 1 and events[0][0] == "request_error" + assert events[0][1]["status"] == 503 + assert events[0][1]["code"] == expected + assert "secret-token" not in json.dumps(events) diff --git a/tests/unit/test_platform_admin_rollout.py b/tests/unit/test_platform_admin_rollout.py index 93d3a5eae5..6bf52c711b 100644 --- a/tests/unit/test_platform_admin_rollout.py +++ b/tests/unit/test_platform_admin_rollout.py @@ -116,7 +116,7 @@ def mint(**kw: Any) -> str: assert minted == [] -def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: +def test_cli_admin_mint_refuses_when_the_connection_scope_cannot_open(monkeypatch, capsys) -> None: from server.auth import jwt from server.cli import access @@ -144,6 +144,50 @@ def mint(**kw): assert code == 1 and out == "" and "enrolled scope" in err +def test_cli_admin_mint_refuses_an_unread_fleet(monkeypatch, capsys) -> None: + import asyncio + + from server.auth import jwt, keyring + from server.cli import access + + connected = object() + reads = [] + minted = [] + + async def unread(ch, *, all_deployments): + assert all_deployments + reads.append(ch) + raise ConnectionError("offline") + + def mint(**kwargs): + minted.append(kwargs) + return "secret-token" + + monkeypatch.setattr(instances, "read_live", unread) + monkeypatch.setattr(keyring, "load_for_connected_mint", AsyncMock()) + monkeypatch.setattr("server.cli._ch.check_acceptance", AsyncMock()) + monkeypatch.setattr(access, "run_connected", lambda run, **kwargs: asyncio.run(run(connected))) + monkeypatch.setattr(access, "_audit_platform_admin_mint", AsyncMock(return_value=True)) + monkeypatch.setattr(jwt, "mint_token", mint) + code = cli.main( + [ + "mint-token", + "--subject", + "sa:controlplane", + "--org", + "acme", + "--platform-admin", + "--ttl-days", + "7", + ] + ) + out, err = capsys.readouterr() + assert reads == [connected] + assert code == 1 and out == "" and "rollout" in err + assert "secret-token" not in out + err + assert minted == [] + + def test_admin_token_floor_matches_the_physical_migration() -> None: from pathlib import Path From 858c90db94b94dd6b8a77bcb7dfd49925da609b4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 18:51:24 +0000 Subject: [PATCH 103/181] WIP: validate serving cell registration and retain wire error codes Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 2 +- docs/architecture/cells.md | 7 ++++--- docs/operations/cell-attach.md | 5 +++-- server/cli/cell_attach.py | 6 ++++-- server/cli/cell_control.py | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 3753ab4f90..e4a436b2ce 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -957,7 +957,7 @@ async def _memory_set( if response is not None: try: envelope = response.json() - candidate = envelope.get("error", {}).get("code", "") + candidate = envelope.get("detail", {}).get("code", "") if isinstance(candidate, str) and re.fullmatch( r"[A-Z][A-Z0-9_]{0,63}", candidate ): diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 13a673223f..7038121a79 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1168,9 +1168,10 @@ before the wait is judged spent, so `--wait 0` is one try of each wait with no r `--scan-budget` (default 2 hours) bounds each full scan, which `--wait` does not count. `--dry-run` runs the refusals and the scan, prints what would be placed and skipped and the scan's time, and writes nothing; the router's checks at `attach/begin` do not run. On a -`serving` row attach reads the cell's mode: `enforced` is done (exit 0), and any other mode -refuses: the directory and the cell's Postgres disagree, as after a restore of one of them, -and attach lowers and repairs neither. +`serving` row attach first verifies that the connected authority, configured cell and +bucket match the directory registration. It then reads the cell's mode: `enforced` is +done (exit 0), and any other mode refuses: the directory and the cell's Postgres disagree, +as after a restore of one of them, and attach lowers and repairs neither. A later cell is attached only once the first is `serving`. It must hold no org data (attach refuses otherwise) and starts in `enforced`. Its attach runs, in order: the diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 20b541aec3..53e176d1cd 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -99,8 +99,9 @@ does not establish that server locks or an unacknowledged commit have disappeare Retry the command to resolve an unknown commit through its durable state checks. A refusal names what to fix. Rerun attach after fixing it: every step is idempotent, and a -rerun resumes by the directory row and the cell's mode. On a `serving` row it prints that it -is done. +rerun resumes by the directory row and the cell's mode. On a `serving` row it verifies +that the connected authority, configured cell and bucket match the directory registration, +then requires `enforced` mode before printing that it is done. ## The point of no return diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 821a77448b..ea20827a35 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -6,7 +6,8 @@ goes, each step idempotent, so a rerun resumes by the directory row and the cell's mode. Steps 1 to 4 are the design's, as cells.md numbers them: -- the directory row: on `serving` it is done (exit 0) once the cell reads `enforced`; +- the directory row: on `serving` it is done (exit 0) once the connected authority and + configured cell and bucket match the registration and the cell reads `enforced`; - the refusals, before any write: the stores are the registered ones, the session is direct and marked, the cell's processes (their heartbeats), the roles, settings and subscriptions that would let an unchecked writer through, the ClickHouse writers, the @@ -560,10 +561,11 @@ async def serving(self) -> dict[str, Any]: # -- the run ----------------------------------------------------------------------- async def done(self) -> bool: - """On a `serving` row: True once the cell reads `enforced`; Refused otherwise.""" + """Require matching Postgres and cell settings, then `enforced` mode.""" from server.core import pg async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: + await cell_control.paired(conn, self.row) state, _ = await apg.read_mode(conn) if state != "enforced": raise _refused( diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index bf412e2d08..93cc7e2e6b 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -58,7 +58,7 @@ async def check_lock(conn: Any) -> None: async def paired(conn: Any, row: dict[str, Any]) -> None: - """The locked authority and configured bucket name the observed registration.""" + """The connected authority and configured bucket name the observed registration.""" recorded = await conn.fetchval("SELECT deployment_id FROM loom.authority") prefix = settings.document_s3_prefix if settings.document_s3_bucket else "" if ( @@ -68,7 +68,7 @@ async def paired(conn: Any, row: dict[str, Any]) -> None: or (settings.document_s3_bucket, prefix) != (row.get("bucket"), row.get("bucket_prefix")) ): raise _refused( - "this cell configuration and locked Postgres do not match the directory registration" + "this cell configuration and connected Postgres do not match the directory registration" ) From 2a379d0d34cb2a9010944af869f09c6fd4981263 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:04:15 +0000 Subject: [PATCH 104/181] Native command cleanup has a configured deadline Keep the existing drain allowance as a positive finite setting. Exercise current command and shutdown boundaries and isolate permanent retirement state between tests. Signed-off-by: zlareb1 --- docs/reference/configuration.md | 1 + loom.example.yaml | 4 ++ schemas/loom-settings.schema.json | 6 +++ server/insights/app/provider_command.py | 4 +- server/settings/catalog.py | 3 ++ tests/integration/test_cell_control_fence.py | 2 +- tests/unit/test_cell_retirement.py | 1 + tests/unit/test_cli_native_completion.py | 2 +- tests/unit/test_cli_native_first_error.py | 4 +- tests/unit/test_provider_command.py | 10 ++++ tests/unit/test_readiness.py | 48 +++++++++++++++++--- tests/unit/test_settings_overrides.py | 7 ++- 12 files changed, 77 insertions(+), 15 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2a78730ac4..81d73f8c9e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1738,6 +1738,7 @@ Process shape, deployment, event stream and cost budgets. | `LOOM_KEEPER_SWEEP_INTERVAL_SECONDS` | `float` | `3600.0` | | Seconds between passes of the always-on sweep that deletes the KeeperMap keys (leases, claims, graph slots) a stopped process left behind. | | `LOOM_WORKERS_PROCESS` | `'child' \| 'inline'` | `'child'` | | auto mode: where the lease holder runs the workers. child = a separate process it starts and stops with the lease, so the passes do not share the event loop that serves requests; inline = in the serving process. | | `LOOM_SHUTDOWN_READY_DELAY_S` | `float` | `15.0` | | Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so the load balancer stops routing to it before its port closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. | +| `LOOM_NATIVE_COMMAND_DRAIN_TIMEOUT_S` | `float` | `15.0` | | Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired. | | `LOOM_BOOT_WARMUP_TIMEOUT_S` | `float` | `30.0` | | Seconds boot waits for the first load of the tracker registry and the runtime overrides, and the first publish of the dashboard bundle, before it serves; readiness stays false until the first two load. | | `LOOM_READY_REQUIRES_MIRRORS` | `bool` | `False` | | True: /v1/health/ready also waits until the mirror warm-up has queued its clones and while a clone of this host's git mirrors is running or queued (a failed clone does not hold it). Needs LOOM_CODE_MIRROR_WARMUP: without the warm-up nothing is cloned at boot and readiness does not wait for mirrors. Off by default: history ops answer `no_git_mirror` meanwhile. | | `LOOM_RUNTIME_OVERRIDES_REFRESH_SECONDS` | `float` | `10.0` | | Seconds between re-reads of loom.runtime_overrides (the admin rate-limit, latency-budget and live-setting overrides) in every process. | diff --git a/loom.example.yaml b/loom.example.yaml index 44d36aece7..17df27d780 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -4891,6 +4891,10 @@ settings: # probe interval × failure threshold. 0 = stop at once. [restart] # shutdown_ready_delay_s: 15.0 + # Seconds a native CLI command allows its producers, deferred writes and participant + # retirement to finish. Expiry leaves the participant unretired. [restart] + # native_command_drain_timeout_s: 15.0 + # Seconds boot waits for the first load of the tracker registry and the runtime # overrides, and the first publish of the dashboard bundle, before it serves; # readiness stays false until the first two load. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index fdbd311ef2..7e4b5d50a0 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -5324,6 +5324,12 @@ "type": "number", "description": "loom.namespace_write_marks is an in-process max per (org, namespace, plane), flushed as one INSERT this often." }, + "native_command_drain_timeout_s": { + "default": 15.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired." + }, "oauth_access_token_ttl_seconds": { "default": 3600, "maximum": 86400, diff --git a/server/insights/app/provider_command.py b/server/insights/app/provider_command.py index dd16904ddb..6c91b93f0a 100644 --- a/server/insights/app/provider_command.py +++ b/server/insights/app/provider_command.py @@ -17,8 +17,6 @@ log = logging.getLogger(__name__) -COMMAND_DRAIN_S = 15.0 # bounded cleanup; timeout leaves the participant unretired - async def _adopt_overrides(*, failing: bool = False) -> bool: """Whether the stored overrides loaded. A failure is logged when a run of them @@ -64,7 +62,7 @@ async def _finish_participant() -> None: from server.memory.app import embedder_calibration loop = asyncio.get_running_loop() - end = loop.time() + COMMAND_DRAIN_S + end = loop.time() + settings.native_command_drain_timeout_s first = gate.producer_failure stopped = False try: diff --git a/server/settings/catalog.py b/server/settings/catalog.py index 8d9117b934..ed30d3a68f 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5145,6 +5145,9 @@ class RuntimeSettings(BaseModel): # answers 503, so the load balancer stops routing to it before its port # closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. shutdown_ready_delay_s: float = Field(default=15.0, ge=0) + # Seconds a native CLI command allows its producers, deferred writes and + # participant retirement to finish. Expiry leaves the participant unretired. + native_command_drain_timeout_s: float = Field(default=15.0, gt=0, allow_inf_nan=False) # Seconds boot waits for the first load of the tracker registry and the # runtime overrides, and the first publish of the dashboard bundle, before # it serves; readiness stays false until the first two load. diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e14eab1220..2e45540fb4 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -47,7 +47,7 @@ async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: transport = httpx.ASGITransport(app=app.build_app()) async with httpx.AsyncClient( transport=transport, - base_url="http://router", + base_url="http://test", headers={"Authorization": f"Bearer {TOKEN}"}, ) as client: yield client, conn, own diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index 178ba2bd1f..f3e7593fd3 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -62,6 +62,7 @@ async def write(proof): entered.set() await finish.wait() + monkeypatch.setattr(barrier, "_OWN", barrier._Own()) monkeypatch.setattr(barrier, "record_retirement", write) monkeypatch.setattr(barrier, "drain_task_sets", AsyncMock(return_value=True)) monkeypatch.setattr(barrier, "flush_registered_buffers", AsyncMock(return_value=True)) diff --git a/tests/unit/test_cli_native_completion.py b/tests/unit/test_cli_native_completion.py index 6b27052577..a2ff54b234 100644 --- a/tests/unit/test_cli_native_completion.py +++ b/tests/unit/test_cli_native_completion.py @@ -56,7 +56,7 @@ async def work(_): def test_permanent_heartbeat_respects_same_absolute_native_allowance( native, monkeypatch, first_class ): - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) monkeypatch.setattr( barrier, "_HEARTBEAT", SimpleNamespace(signal_stop=lambda: None, running=True) ) diff --git a/tests/unit/test_cli_native_first_error.py b/tests/unit/test_cli_native_first_error.py index c77d29ba83..3fad11815a 100644 --- a/tests/unit/test_cli_native_first_error.py +++ b/tests/unit/test_cli_native_first_error.py @@ -353,7 +353,7 @@ async def work(_): def test_unfinished_role_producer_prevents_success_and_certificate(native, monkeypatch): producers = [] - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0) async def work(_): async def producer(): @@ -399,7 +399,7 @@ async def producer(error): def test_actual_retirement_refusal_stays_registered_and_refuses_success(native, monkeypatch): - monkeypatch.setattr(command, "COMMAND_DRAIN_S", 0.02) + monkeypatch.setattr(command.settings, "native_command_drain_timeout_s", 0.02) stopped = [] monkeypatch.setattr( barrier, diff --git a/tests/unit/test_provider_command.py b/tests/unit/test_provider_command.py index 9a70d03b37..645b0fcb3e 100644 --- a/tests/unit/test_provider_command.py +++ b/tests/unit/test_provider_command.py @@ -8,6 +8,16 @@ from server.insights.app import provider_command as command +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +def test_native_drain_budget_is_finite_and_positive(value): + from pydantic import ValidationError + + from server.settings.catalog import RuntimeSettings + + with pytest.raises(ValidationError): + RuntimeSettings(native_command_drain_timeout_s=value) + + @pytest.fixture def setup(monkeypatch): from server.core import retirement diff --git a/tests/unit/test_readiness.py b/tests/unit/test_readiness.py index b7adc50935..49e153bc68 100644 --- a/tests/unit/test_readiness.py +++ b/tests/unit/test_readiness.py @@ -441,16 +441,50 @@ async def insert(self, table, rows, *, settings=None): usage._reset_for_tests() -def test_shutdown_drains_share_a_budget_before_retirement() -> None: +@pytest.mark.asyncio +async def test_shutdown_drains_share_a_budget_before_retirement(monkeypatch) -> None: import inspect + from types import SimpleNamespace from server import main - - src = inspect.getsource(main._shut_down) - budget = src.index("_shutdown_drains(lease, supervisor, emitter), timeout=left") - assert ( - src.index("await _instances.stop(ch)") < budget < src.index("await boot.leave_deployment(") - ) + from server.core import instances + from server.insights.app.provider_barrier import SHUTDOWN_RETIRE_S + + clock = [100.0] + calls = [] + wait_timeouts = [] + ch, lease, supervisor, emitter = object(), object(), object(), object() + + async def stop(client): + assert client is ch + calls.append("stop") + clock[0] += 3.0 + + async def drains(actual_lease, actual_supervisor, actual_emitter): + assert (actual_lease, actual_supervisor, actual_emitter) == (lease, supervisor, emitter) + calls.append("drains") + clock[0] += 2.0 + + async def wait(tasks, **kwargs): + wait_timeouts.append(kwargs["timeout"]) + return await asyncio.wait(tasks, **kwargs) + + async def leave(client, *, budget_s): + assert client is ch + calls.append("retire") + assert budget_s == SHUTDOWN_RETIRE_S + 5.0 + + monkeypatch.setattr(main, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(main, "asyncio", SimpleNamespace(wait=wait)) + monkeypatch.setattr(instances, "stop", stop) + monkeypatch.setattr(main, "_shutdown_drains", drains) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(boot, "SHUTDOWN_DRAIN_BUDGET_S", 10.0) + + await main._shut_down(ch, lease, supervisor, emitter) + + assert calls == ["stop", "drains", "retire"] + assert wait_timeouts == [7.0] life = inspect.getsource(main.lifespan) assert "await _shut_down(" in life and "ch.disconnect()" not in life diff --git a/tests/unit/test_settings_overrides.py b/tests/unit/test_settings_overrides.py index e8eabb6761..e14c88c9c2 100644 --- a/tests/unit/test_settings_overrides.py +++ b/tests/unit/test_settings_overrides.py @@ -453,13 +453,18 @@ async def test_a_native_command_chunks_as_the_http_op_does_under_an_override( ) -> None: """`loom rechunk` and knowledge.upsert share `_chunk_text`: inside a native command it reads the stored override, not the startup default.""" + from server.core import retirement from server.insights.app import provider_command as pc from server.knowledge.app import writes async def _noop(*_a: Any, **_k: Any) -> bool: return True - for name in ("register_process", "load_active_configuration", "retire_process"): + async def _register() -> None: + retirement.get_gate().enrolled = True + + monkeypatch.setattr(pc, "register_process", _register) + for name in ("load_active_configuration", "_retire_until"): monkeypatch.setattr(pc, name, _noop) monkeypatch.setattr(pc, "drain_task_sets", _noop) monkeypatch.setattr(pc, "flush_registered_buffers", _noop) From 300c66578e9af5dfd4fa2fc801553c5d3708ec02 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:14:40 +0000 Subject: [PATCH 105/181] Check repeated cancellation against the original clock deadline Signed-off-by: zlareb1 --- tests/unit/test_cell_place_owner.py | 54 +++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py index 2cce1d25cf..b8dac073fb 100644 --- a/tests/unit/test_cell_place_owner.py +++ b/tests/unit/test_cell_place_owner.py @@ -4,12 +4,14 @@ import asyncio import time +from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from server.cli import cell_control from server.core import pg +from tests.unit.clock_fake import ManualClock from tests.unit.test_cell_attach import FakeConn pytestmark = pytest.mark.asyncio @@ -163,7 +165,11 @@ def is_closed(self): await asyncio.gather(task, return_exceptions=True) -async def test_repeated_signals_cannot_extend_an_expired_public_close(): +async def test_repeated_signals_cannot_extend_an_expired_public_close(monkeypatch): + clock = ManualClock(t=100.0) + local_time = SimpleNamespace(monotonic=clock.monotonic) + monkeypatch.setattr(cell_control, "time", local_time) + monkeypatch.setattr(pg, "time", local_time) closing = asyncio.Event() conn = FakeConn() @@ -174,28 +180,46 @@ async def stalled(**kwargs): conn.close = stalled owner = cell_control.Owner(exit_s=0.04) owner.conn = conn - started = time.monotonic() - task = asyncio.create_task(owner.close()) - await closing.wait() - end = owner.close_end - - async def signals(): - while not task.done(): - task.cancel("first signal") - await asyncio.sleep(0.001) + waits = [] - signal_task = asyncio.create_task(signals()) + async def signalled_wait(tasks, **kwargs): + assert tasks == {owner._close_task} + waits.append(kwargs["timeout"]) + assert len(waits) <= 2, "signals cannot grant another cleanup allowance" + await closing.wait() + if len(waits) == 1: + clock.advance(0.01) + else: + clock.t = 100.04 + asyncio.current_task().cancel("first signal" if len(waits) == 1 else "second signal") + await asyncio.sleep(0) + raise AssertionError("cancellation must reach the public close") + + # Replace only this module's wait seam; native close and the event loop stay real. + monkeypatch.setattr( + cell_control, + "asyncio", + SimpleNamespace( + create_task=asyncio.create_task, + CancelledError=asyncio.CancelledError, + wait=signalled_wait, + ), + ) + task = asyncio.create_task(owner.close()) try: done, _ = await asyncio.wait({task}, timeout=0.2) assert done, "repeated signals cannot restart the original cleanup allowance" with pytest.raises(asyncio.CancelledError, match="first signal"): await task - assert time.monotonic() - started < 0.2 - assert owner.close_end == end + assert waits == pytest.approx([0.04, 0.03]) + assert clock.monotonic() - 100.0 == pytest.approx(0.04) + assert owner.close_end == 100.04 assert "TERMINATE" in conn.executed finally: - signal_task.cancel() - await asyncio.gather(signal_task, task, return_exceptions=True) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + if owner._close_task is not None: + await asyncio.gather(owner._close_task, return_exceptions=True) async def test_closed_transport_cannot_acknowledge_pending_public_cancellation(): From c7efa71b4d68da03da2c4d9ec1f13be8e829b2a1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:18:14 +0000 Subject: [PATCH 106/181] Format the cell cleanup deadline test Signed-off-by: zlareb1 --- tests/unit/test_cell_place_owner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_cell_place_owner.py b/tests/unit/test_cell_place_owner.py index 89fb7571ed..08efc60bb6 100644 --- a/tests/unit/test_cell_place_owner.py +++ b/tests/unit/test_cell_place_owner.py @@ -239,7 +239,6 @@ async def main(): assert "original-after-public-close" in result.stdout - async def test_repeated_signals_cannot_extend_an_expired_public_close(monkeypatch): clock = ManualClock(t=100.0) local_time = SimpleNamespace(monotonic=clock.monotonic) From 8444e8d82a792907e4dcaab919ceb14edcc15514 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:27:56 +0000 Subject: [PATCH 107/181] Check that native participant censuses exclude certified history Signed-off-by: zlareb1 --- .../test_provider_coordination_live_ch.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/integration/test_provider_coordination_live_ch.py b/tests/integration/test_provider_coordination_live_ch.py index 16c9704fa8..8e74b88368 100644 --- a/tests/integration/test_provider_coordination_live_ch.py +++ b/tests/integration/test_provider_coordination_live_ch.py @@ -1,6 +1,8 @@ """Real participant rows and Keeper recovery on an opt-in disposable database.""" +import contextlib import os +import sys from uuid import uuid4 import pytest @@ -58,6 +60,78 @@ async def test_participant_ack_is_full_row_and_expiry_is_on_the_server_clock(mon await ch.disconnect() +@pytest.mark.asyncio +async def test_participant_census_excludes_certified_history_before_returning_rows(): + _live() + from server.core.ch import query_tag, sequential_reads + from server.core.provider_process import certified, has_certificate + from server.insights.store import provider_participants as store + + legacy = "test|boot||pidns|7|99||" + proof = certified(legacy) + invalid_pid = proof.replace("|7|99|", "|0|99|") + oversized_pid = proof.replace("|7|99|", "|" + "5" * 5000 + "|99|") + assert has_certificate(proof) + assert not has_certificate(invalid_pid) + assert not has_certificate(oversized_pid) + blockers = { + uuid4(): (state, identity) + for state, identity in ( + ("running", proof), + ("drained", proof), + ("retired", ""), + ("retired", legacy), + ("retired", legacy + "|unknown"), + ("retired", proof + "|unknown"), + ("retired", invalid_pid), + ("retired", oversized_pid), + ) + } + history = {uuid4(): ("retired", proof) for _ in range(256)} + created = {**history, **blockers} + expected = {str(identity): value for identity, value in blockers.items()} + ids = [str(identity) for identity in created] + ch = _default_client() + with query_tag("test.provider_participant_census"): + await ch.connect() + try: + await ch.insert( + store.TABLE, + [ + {"participant_id": identity, "state": state, "version": 1, "host_proof": value} + for identity, (state, value) in created.items() + ], + ) + with sequential_reads(): + raw = await ch.query_all( + "SELECT participant_id, state, host_proof FROM (" + + store.PARTICIPANTS_SQL + + ") WHERE participant_id IN {ids:Array(UUID)}", + {"ids": ids}, + ) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) for row in raw + } == expected, "ClickHouse must exclude certified history before returning census rows" + kept = await store.participants(ch=ch) + assert { + str(row["participant_id"]): (row["state"], row["host_proof"]) + for row in kept + if str(row["participant_id"]) in ids + } == expected + finally: + with ( + contextlib.suppress(BaseException) if sys.exception() else contextlib.nullcontext() + ): + try: + await ch.execute( + f"ALTER TABLE {store.TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}} " + "SETTINGS mutations_sync=2", + {"ids": ids}, + ) + finally: + await ch.disconnect() + + @pytest.mark.asyncio async def test_keeper_never_steals_from_a_beating_process_even_with_expired_timestamp(): _live() From 11f7b462731334dbb01d8caaeccaaa00f408b49d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:40:24 +0000 Subject: [PATCH 108/181] Document update control writers and nonce cleanup limits Signed-off-by: zlareb1 --- docs/architecture/cells.md | 34 +++++++++++++++++++-------------- docs/operations/cell-update.md | 6 +++++- server/cli/cell_update_probe.py | 2 +- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index f64bf8e011..a2cfe31ba3 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1379,20 +1379,26 @@ The source census for acceptance and operator directory writes is checked by `tests/unit/test_cell_control_census.py`, including dynamic cleanup tables, guarded entry points and the named exceptions: -| Writer | Fence | -| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | -| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | -| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | -| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | -| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | -| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | -| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | - -Read-only routes are exempt. Router migrations and an offline directory restore are -maintenance operations. Future restore/update acceptance writers must use the same -claim and lock-session helpers; a new directory mutation must use the common comparison -before its no-op branch as well. Direct privileged SQL is not an online control writer. +| Writer | Fence | +| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cell_prepare._write_rows` | Fresh claim, observed local rows, exact lock backend | +| `cell_attach_pg._mode_transaction` | Attach's claim, supervised mode try, recording/first/fingerprint comparison, exact lock backend | +| `cell_control.backout` | Fresh claim, fenced close and zero-placement read, observed local rows, exact lock backend | +| `cell_update_pg.write.change` | Fresh directory claim; owner's bounded transaction on the exact ATTACH backend, paired registration and exact observed local identity/pair before tenancy repair and acceptance write | +| `directory.attach_begin`, `attach_place`, `attach_enforced`, `attach_serving`, `remove` | Claimed generation, full registration and exact pair, including retries/no-ops | +| `directory.update_addresses` | Claimed generation, full registration and exact pair under directory/row locks; commit-time address check, audit and synchronous WAL acknowledgement in one bounded transaction | +| `directory.update_configuration`, `_configuration_commit` | Claimed generation, full registration and exact pair under ordered org/directory/row locks; root-secret and reservation checks, placements, audit and acceptance commit in one bounded synchronous transaction | +| `directory.configuration_placements` | Claimed generation, full registration and exact pair under directory/row locks; root-secret and actual winner checks with audit and synchronous WAL acknowledgement in one bounded transaction | +| `directory.set_limit`, `rotate_credential` | Read and compare the current generation/registration/pair; do not claim or rotate | +| `directory.register` | Bootstrap default or immutable rerun lookup; never replaces existing acceptance or generation | +| `directory.place` | Immutable placement only; keeps its org/cell locks and quota checks; never writes local acceptance | + +Ordinary read-only routes are exempt; `configuration_placements` is an audited, +WAL-fenced proof before local tenancy repair. Router migrations and an offline +directory restore are maintenance operations. Update acceptance writers use the +same claim and lock-session helpers; any additional acceptance writer must do so +as well. A directory mutation uses the common comparison before its no-op branch. +Direct privileged SQL is not an online control writer. The [control upgrade and recovery procedure](../operations/cell-control.md) defines the required fleet turnover and offline restore of generations. N-1 data-plane builds can read and insert on the additive schema; old control clients and router handlers cannot diff --git a/docs/operations/cell-update.md b/docs/operations/cell-update.md index 1a185b61df..b697c6311b 100644 --- a/docs/operations/cell-update.md +++ b/docs/operations/cell-update.md @@ -140,4 +140,8 @@ routing table indefinitely. The command cannot certify all-router agreement. Configuration actions, finish, revert and probe refresh are mutually exclusive. Probe readiness retries use `--wait`; invalid proofs refuse immediately. Nonce -cleanup compares this run's values, leaving another run's nonce untouched. +cleanup uses a value-qualified Postgres delete and a Keeper compare-and-delete. +Bucket cleanup reads and compares this run's value before an unconditional delete. +If the ATTACH backend is lost, a successor can replace the object between that read +and delete and lose its nonce. Rerun the affected proof; bucket cleanup does not +guarantee preservation of a successor's nonce. diff --git a/server/cli/cell_update_probe.py b/server/cli/cell_update_probe.py index 7d3dbe2902..d102969225 100644 --- a/server/cli/cell_update_probe.py +++ b/server/cli/cell_update_probe.py @@ -1,4 +1,4 @@ -"""Fresh store proof using the existing bounded probe retries and exact nonce cleanup.""" +"""Fresh store proof with bounded retries and prepare's store-specific nonce cleanup.""" from __future__ import annotations From 77b0638a4a199f60929d43725a0792a6b497ef67 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:40:57 +0000 Subject: [PATCH 109/181] SCIM person authorization checks active identities across requests Preserve the reviewed person, session and erasure authorization work before composing the cell attachment foundation. Signed-off-by: zlareb1 --- .../migrations/101_oauth_verified_email.sql | 4 + server/assistant/app/routes.py | 6 +- server/auth/authz.py | 73 ++- server/auth/bearer.py | 10 +- server/auth/device.py | 53 ++- server/auth/jwt.py | 18 + server/auth/machines.py | 1 + server/auth/oauth.py | 80 +++- server/auth/oauth_store.py | 25 +- server/auth/routes.py | 6 +- server/auth/scim.py | 31 +- server/auth/scim_people.py | 306 ++++++++++++ server/auth/self_signup.py | 98 ++-- server/auth/service_accounts.py | 58 ++- server/auth/sso_links.py | 33 +- server/cli/hygiene.py | 4 +- server/code/app/index/registry.py | 15 +- server/code/app/repo_admin.py | 5 +- server/erasure/app/saga_steps.py | 12 +- server/health.py | 8 +- server/identity/app/account_admin.py | 10 +- server/identity/app/agent_claim.py | 11 +- server/identity/app/claim_release.py | 4 +- server/identity/app/collections.py | 10 +- server/identity/app/conversation_logging.py | 4 +- server/identity/app/me_callers.py | 54 +-- server/identity/app/me_routes.py | 4 +- server/identity/app/me_service_accounts.py | 117 +++-- server/identity/app/namespace.py | 10 +- server/identity/app/namespace_autoregister.py | 5 +- server/identity/app/namespace_snapshot.py | 49 +- server/identity/app/password_link.py | 16 +- server/identity/app/scim_groups.py | 4 +- server/insights/app/admin_routes.py | 10 +- server/knowledge/app/download_routes.py | 2 +- server/knowledge/app/upload_routes.py | 2 +- server/tasks/app/watch_routes.py | 6 +- server/tracker/app/registry.py | 56 ++- server/transport/gates.py | 10 +- server/transport/http.py | 6 +- server/transport/mcp.py | 8 +- server/transport/mcp_auth.py | 6 +- server/transport/tenancy_gate.py | 12 +- tests/integration/scim_session_previous.py | 76 +++ .../integration/test_scim_erasure_live_ch.py | 292 ++++++++++++ tests/integration/test_scim_person_live_ch.py | 328 +++++++++++++ .../integration/test_scim_sessions_live_ch.py | 279 +++++++++++ tests/unit/mcp_probe.py | 3 +- tests/unit/test_admission_gates.py | 3 +- tests/unit/test_admission_parity.py | 3 +- tests/unit/test_assistant_bridge.py | 29 +- tests/unit/test_auth_routes.py | 3 +- tests/unit/test_bearer_parse.py | 8 +- tests/unit/test_ch_caller_shaped.py | 2 +- tests/unit/test_cross_org_rule.py | 5 +- tests/unit/test_delegation.py | 4 +- tests/unit/test_dev_mode_token_owner.py | 76 ++- .../unit/test_http_idempotency_after_auth.py | 3 +- .../test_idempotency_across_transports.py | 3 +- tests/unit/test_op_transport_contract.py | 11 +- .../unit/test_ratelimit_per_org_namespace.py | 3 +- .../unit/test_raw_routes_restricted_audit.py | 5 +- tests/unit/test_request_sli.py | 23 +- ...st_revocation_fails_closed_until_loaded.py | 19 +- tests/unit/test_scim_admission.py | 370 +++++++++++++++ tests/unit/test_scim_cell_authorization.py | 9 +- .../unit/test_scim_erasure_identity_writes.py | 291 ++++++++++++ tests/unit/test_scim_fresh_reader_bounds.py | 55 +++ tests/unit/test_scim_groups.py | 7 +- tests/unit/test_scim_identity_writes.py | 173 +++++++ tests/unit/test_scim_input_bounds.py | 57 +++ tests/unit/test_scim_people.py | 444 ++++++++++++++++++ tests/unit/test_scim_sessions.py | 412 ++++++++++++++++ tests/unit/test_sharing.py | 23 +- tests/unit/test_task_watch.py | 5 +- tests/unit/test_token_mint.py | 5 +- tests/unit/test_trusted_caller.py | 30 +- 77 files changed, 3892 insertions(+), 429 deletions(-) create mode 100644 schemas/migrations/101_oauth_verified_email.sql create mode 100644 server/auth/scim_people.py create mode 100644 tests/integration/scim_session_previous.py create mode 100644 tests/integration/test_scim_erasure_live_ch.py create mode 100644 tests/integration/test_scim_person_live_ch.py create mode 100644 tests/integration/test_scim_sessions_live_ch.py create mode 100644 tests/unit/test_scim_admission.py create mode 100644 tests/unit/test_scim_erasure_identity_writes.py create mode 100644 tests/unit/test_scim_fresh_reader_bounds.py create mode 100644 tests/unit/test_scim_identity_writes.py create mode 100644 tests/unit/test_scim_input_bounds.py create mode 100644 tests/unit/test_scim_people.py create mode 100644 tests/unit/test_scim_sessions.py diff --git a/schemas/migrations/101_oauth_verified_email.sql b/schemas/migrations/101_oauth_verified_email.sql new file mode 100644 index 0000000000..4caed7425d --- /dev/null +++ b/schemas/migrations/101_oauth_verified_email.sql @@ -0,0 +1,4 @@ +-- Carry the verified sign-in address through refresh rotations. +-- Existing rows and writers that omit this field carry no email proof. +ALTER TABLE loom.oauth_state + ADD COLUMN IF NOT EXISTS verified_email String DEFAULT ''; diff --git a/server/assistant/app/routes.py b/server/assistant/app/routes.py index fabf8f4eab..35541b05ca 100644 --- a/server/assistant/app/routes.py +++ b/server/assistant/app/routes.py @@ -70,8 +70,8 @@ def _refusal(e: BaseException) -> LoomServerError | HTTPException: return AuthInsufficientError("the caller may not ask the assistant here") -def _bearer(request: Request) -> AuthInfo: - auth = verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) +async def _bearer(request: Request) -> AuthInfo: + auth = await verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) if auth is None: if not settings.auth_actually_required(): # Each answer reads under a token delegated from the caller's. @@ -211,7 +211,7 @@ async def chat_completions(request: Request) -> Any: """One answer from the Loom assistant. Body: OpenAI's chat request (`messages`, `stream`); any `model`, `conversation_id`, system message, tool or file is ignored. The caller needs `viewer` on the namespace.""" - bearer = _bearer(request) + bearer = await _bearer(request) # An org-wide token's `*` is no org a row can name, as the op transport keeps it out. org_seen = "-" if bearer.org == "*" else bearer.org row = _Row(bearer, org_seen, request.headers.get("X-Loom-Namespace") or "") diff --git a/server/auth/authz.py b/server/auth/authz.py index f1d34e9838..59a40c95ca 100644 --- a/server/auth/authz.py +++ b/server/auth/authz.py @@ -18,7 +18,8 @@ import logging import time -from collections.abc import Awaitable, Callable, Iterable +from collections.abc import Awaitable, Callable, Hashable, Iterable, Iterator +from contextlib import contextmanager from enum import Enum from typing import TYPE_CHECKING @@ -83,6 +84,23 @@ class ActedForRefused(PermissionError): access to an index, so a transport adds no index reason to it.""" +async def _check_provisioning(auth: AuthInfo | None, org: str | None) -> None: + from server.auth import login, scim + from server.core.errors import BackendUnavailableError + + if auth is None: + return + scope_org = auth.org if org is None else org + if not may_act_in_org(auth, scope_org): + raise PermissionError("token is for another org") + try: + refused = await scim.refuses_auth(auth, org=scope_org) + except login.SignInUnavailable as e: + raise BackendUnavailableError("SCIM provisioning cannot be checked yet") from e + if refused: + raise ActedForRefused(_ACTED_FOR_REVOKED) + + async def check_acted_for(auth: AuthInfo | None, org: str | None) -> None: """Refuse a trusted caller acting for a revoked or SCIM-deactivated subject under any spelling it holds, or for an `sa:` subject that is not an active bot or one of its @@ -93,6 +111,7 @@ async def check_acted_for(auth: AuthInfo | None, org: str | None) -> None: from server.auth.gateway import is_end_user from server.core import grant_cache + await _check_provisioning(auth, org) # An end user's token names the user, whose revocation verify checked. if auth is None or not auth.impersonated_by or is_end_user(auth): return @@ -112,14 +131,43 @@ async def check_acted_for(auth: AuthInfo | None, org: str | None) -> None: for email in {s.lower() for s in spellings if "@" in s}: if await _remembered(("scim", email), _scim_deactivated): raise ActedForRefused(_ACTED_FOR_REVOKED) - if bot and await _remembered(("bot", bot), _bot_ended): + from server.auth import scim + + if bot and not scim.requires_provisioning() and await _remembered(("bot", bot), _bot_ended): + await _check_personal_acted_for(org or auth.org, auth.subject, bot) + + +async def _check_personal_acted_for(org: str, subject: str, bot: str) -> None: + from server.auth import service_accounts + from server.core.ch import get_client, own_write_reads + from server.core.datetimes import as_aware_utc + from server.core.errors import BackendUnavailableError + + try: + with own_write_reads(): + machine = await get_client().query_one( + "SELECT active FROM loom.machine_identities FINAL " + "WHERE name = {name:String} LIMIT 1", + params={"name": bot}, + ) + row = None if machine is not None else await service_accounts._creator_row(org, subject) + except Exception as e: # unread status grants no acted-for authority + raise BackendUnavailableError("cannot read the acted-for service account") from e + if machine is not None: + if machine["active"]: + return + raise ActedForRefused(_ACTED_FOR_NO_BOT) + if row is None or not row["created_by"] or row["revoked_at"] is not None: + raise ActedForRefused(_ACTED_FOR_NO_BOT) + expiry = row.get("expires_at") + if expiry is not None and as_aware_utc(expiry).timestamp() <= time.time(): raise ActedForRefused(_ACTED_FOR_NO_BOT) # (kind, key) -> (read at, ended), kept grant_cache_ttl_seconds: a SCIM deactivation, or # a bot created or disabled, by another process reaches this one within that, and one # written here drops its entry (`forget_ended`); emptied past _ENDED_MAX. -_ENDED: dict[tuple[str, str], tuple[float, bool]] = {} +_ENDED: dict[tuple[Hashable, ...], tuple[float, bool]] = {} _forgets = 0 # forgets so far: a read that one overtook is not kept _ENDED_MAX = 10_000 # not a setting: a memory bound on a memo the TTL already ages @@ -129,10 +177,23 @@ def forget_ended(kind: str, key: str) -> None: just wrote, so its own write takes effect here at once.""" global _forgets _ENDED.pop((kind, key.lower()), None) + for decision in tuple(_ENDED): + if decision[0] == "scim_people": + _ENDED.pop(decision, None) _forgets += 1 -async def _remembered(key: tuple[str, str], read: Callable[[str], Awaitable[bool]]) -> bool: +@contextmanager +def identity_write() -> Iterator[None]: + """Do not memoize person checks started before or during an identity write.""" + forget_ended("identity", "") + try: + yield + finally: + forget_ended("identity", "") + + +async def _remembered(key: tuple[Hashable, ...], read: Callable[[str], Awaitable[bool]]) -> bool: from server.settings import settings now = time.monotonic() @@ -140,7 +201,7 @@ async def _remembered(key: tuple[str, str], read: Callable[[str], Awaitable[bool if hit is not None and now - hit[0] < settings.grant_cache_ttl_seconds: return hit[1] forgets = _forgets - ended = await read(key[1]) + ended = await read(str(key[-1])) if _forgets != forgets: return ended # a write here overtook the read: the next check reads afresh if len(_ENDED) >= _ENDED_MAX: @@ -212,6 +273,7 @@ async def check_authorized( # authorize here (the op gate checked before it got this far). from server.auth import tenancy + await check_acted_for(auth, org) await tenancy.ensure_held(org, auth) if auth.creator: @@ -401,6 +463,7 @@ async def _check_creator( raise PermissionError( f"principal {auth.subject!r} (org {auth.org!r}) cannot act in org {account_org!r}" ) + await _check_provisioning(auth, account_org) if not await creator_account_active(account_org, auth.creator): raise PermissionError(f"the account that created this token is not active in {account_org}") if namespace is None and required_role.value >= Role.WRITER.value: diff --git a/server/auth/bearer.py b/server/auth/bearer.py index fc142bb6da..7a6a00deaf 100644 --- a/server/auth/bearer.py +++ b/server/auth/bearer.py @@ -20,7 +20,7 @@ from server.auth.context import quota_key_for from server.auth.gateway import is_end_user -from server.auth.jwt import AuthInfo, AuthUnavailable, TokenError, verify_token +from server.auth.jwt import AuthInfo, AuthUnavailable, TokenError, authenticate_token from server.core.ch import set_quota_key from server.observability.metrics import record_auth_refusal @@ -49,7 +49,7 @@ def missing_bearer() -> HTTPException: ) -def verify_bearer_header( +async def verify_bearer_header( authorization: str | None, *, acts_for_end_users: bool = False ) -> AuthInfo | None: """The principal an `Authorization` header names, None when it carries @@ -64,7 +64,7 @@ def verify_bearer_header( status_code=401, detail={"code": "AUTH_INVALID", "message": str(e)}, ) from e - info = None if token is None else verify_bearer(token) + info = None if token is None else await verify_bearer(token) if info is not None and not acts_for_end_users and is_end_user(info): raise gateway_refused() return info @@ -93,12 +93,12 @@ def auth_unavailable(e: AuthUnavailable) -> HTTPException: ) -def verify_bearer(token: str) -> AuthInfo: +async def verify_bearer(token: str) -> AuthInfo: """The principal `token` names; HTTPException 401 / 503 otherwise. The rest of the request's ClickHouse queries carry its quota key, so routes outside the op registry draw the caller's bucket too.""" try: - info = verify_token(token) + info = await authenticate_token(token) except TokenError as e: record_auth_refusal(e, transport="http") if isinstance(e, AuthUnavailable): diff --git a/server/auth/device.py b/server/auth/device.py index 8bedba0b68..519571aa00 100644 --- a/server/auth/device.py +++ b/server/auth/device.py @@ -19,7 +19,7 @@ from fastapi import APIRouter, Request from fastapi.responses import JSONResponse, Response -from server.auth import attempts, claims, oauth, oidc +from server.auth import attempts, claims, oauth, oidc, scim from server.auth.attempts import Attempts, address_key, client_address from server.auth.jwt import new_jti from server.auth.oauth_store import CodeReplayed, Device, token_id @@ -251,12 +251,22 @@ async def signed_in( user_code: str, identity: tuple[str, str, list[str], list[str]], upstream_refresh: str, + *, + verified_email: str = "", ) -> Response: """The user has signed in: remember who, and ask them to confirm. Two different people signing in with one code means one of them got it from the other's screen; neither can tell which, so the grant ends and the terminal has to start again.""" subject, org, *_ = identity + if refused := await oauth._refused_sign_in( + subject, + {"kind": "device", "user_code": user_code}, + upstream_refresh, + org=org, + verified_email=verified_email, + ): + return refused ticket = secrets.token_urlsafe(32) try: for attempt in range(3): @@ -265,7 +275,9 @@ async def signed_in( outcome, dropped = "decided", [upstream_refresh] break try: - outcome, dropped = await _sign_in(device, identity, upstream_refresh, ticket) + outcome, dropped = await _sign_in( + device, identity, upstream_refresh, ticket, verified_email=verified_email + ) break except claims.Changed: if attempt == 2: @@ -312,6 +324,8 @@ async def _sign_in( identity: tuple[str, str, list[str], list[str]], upstream_refresh: str, ticket: str, + *, + verified_email: str = "", ) -> tuple[str, list[str]]: """Record who signed in on `device`: the outcome, and the IdP tokens no longer needed.""" @@ -324,17 +338,17 @@ async def _sign_in( return "conflict", [upstream_refresh, device["upstream_refresh"]] # The same person signing in again replaces their IdP token. previous = device["upstream_refresh"] - await oauth.store.save_device( - device - | { - "subject": subject, - "org": org, - "namespaces": namespaces, - "roles": roles, - "upstream_refresh": upstream_refresh, - "ticket": token_id(ticket), - } - ) + signed_in_device: Device = { + **device, + "subject": subject, + "org": org, + "namespaces": namespaces, + "roles": roles, + "upstream_refresh": upstream_refresh, + "verified_email": scim.normalized(verified_email), + "ticket": token_id(ticket), + } + await oauth.store.save_device(signed_in_device) return "confirm", [previous] if previous != upstream_refresh else [] @@ -352,6 +366,8 @@ async def deny(user_code: str) -> None: @router.post("/oauth/device/approve") async def device_approve(request: Request) -> Response: + from server.auth.login import SignInUnavailable + f = await oauth.form_body(request) ticket = f.get("ticket", "") approved = f.get("decision") == "approve" @@ -364,6 +380,10 @@ async def device_approve(request: Request) -> Response: or not secrets.compare_digest(device["ticket"], token_id(ticket)) ): return _expired_page() + if approved and await scim.refuses_subject( + device["subject"], org=device["org"], verified_email=device.get("verified_email", "") + ): + approved = False dropped = "" if approved else device["upstream_refresh"] device |= { "status": "approved" if approved else "denied", @@ -372,6 +392,13 @@ async def device_approve(request: Request) -> Response: "upstream_refresh": device["upstream_refresh"] if approved else "", } await oauth.store.save_device(device) + except SignInUnavailable: + return oauth.html_page( + "Sign in to Loom", + "

Sign-in cannot be checked

Go back and submit your choice again.

", + 503, + headers={"Retry-After": "5"}, + ) except claims.Busy: return _busy_page() # a double submit (the other one decided), or Keeper is down await _revoke_all(dropped) diff --git a/server/auth/jwt.py b/server/auth/jwt.py index 9b55f7fa1b..8eeab98ee6 100644 --- a/server/auth/jwt.py +++ b/server/auth/jwt.py @@ -509,6 +509,24 @@ def _end_user_key(token: str) -> tuple[str, EndUserIssuer, Any] | None: ) +async def authenticate_token(token: str, *, secret: str | None = None) -> AuthInfo: + """Verify the credential, then admit its complete person and ancestry graph.""" + from server.auth import login, scim + + auth = verify_token(token, secret=secret) + try: + refused = await scim.refuses_auth(auth) + except login.SignInUnavailable as e: + raise AuthUnavailable( + "SCIM provisioning cannot be checked yet; retry shortly", reason="provisioning" + ) from e + if refused: + raise TokenError( + "the person is not actively provisioned in this cell", reason="provisioning" + ) + return auth + + def _verify_end_user(token: str, issuer: str, spec: EndUserIssuer, public_key: Any) -> AuthInfo: """A short-lived token an end-user issuer signed for one of its users: one org, `sub: user:`, and the agent the request runs inside when there is diff --git a/server/auth/machines.py b/server/auth/machines.py index fcfbd28730..f89609d008 100644 --- a/server/auth/machines.py +++ b/server/auth/machines.py @@ -235,6 +235,7 @@ async def save(m: Machine, *, by: str, active: bool = True, ch: Any = None) -> N # FINAL a later change with a lower version would be lost. ch = ch or get_client() version = await next_version(ch, TABLE, {"name": m["name"]}) + forget_ended("bot", m["name"]) try: await ch.insert( TABLE, diff --git a/server/auth/oauth.py b/server/auth/oauth.py index fb028b1ebf..7300c58cfd 100644 --- a/server/auth/oauth.py +++ b/server/auth/oauth.py @@ -422,11 +422,14 @@ async def _redirect_with_code( upstream_refresh: str, state: str, since: int, + verified_email: str = "", ) -> RedirectResponse: """Mint the MCP client's single-use code for a signed-in user and send the browser back to the client with it. `since` is when the identity was resolved: a subject revoked after it gets no tokens for the code.""" subject, org, namespaces, roles = identity + if await scim.refuses_subject(subject, org=org, verified_email=verified_email): + raise _invalid_grant("this person cannot sign in under the cell's provisioning policy") code = secrets.token_urlsafe(32) await store.save_code( code, @@ -442,6 +445,7 @@ async def _redirect_with_code( "since": since, "family": new_jti(), "upstream_refresh": upstream_refresh, + "verified_email": scim.normalized(verified_email), }, ) return _back_to_client(client["redirect_uri"], {"code": code}, state) @@ -794,8 +798,8 @@ async def _account_for( from server.auth.login import SignInUnavailable, resolve_identity_for_email try: - if await scim.deactivated(email): # before JIT makes them an account - return _Denied(_DEACTIVATED) + if await scim.refuses_subject(email, org="", verified_email=email): # before JIT writes + return _Denied(_SIGN_IN_REFUSED) await finish_unfinished_account(email) resolved = await resolve_identity_for_email(email) if resolved is not None: @@ -814,7 +818,7 @@ async def _account_for( return _sign_in_unavailable(_restart_url(pending)) -_DEACTIVATED = "this account is deactivated" +_SIGN_IN_REFUSED = "this account is not allowed to sign in" async def _signed_in_as( @@ -824,38 +828,43 @@ async def _signed_in_as( email: str, ) -> Response: """Carry on the sign-in `pending` began, as `identity`. Every SSO sign-in - ends here, so here is where a SCIM deactivation is sure to be checked.""" + ends here, so the resolved identity is checked before the next grant.""" from server.auth import device - if refused := await _refused_if_deactivated(email, pending, upstream): + if refused := await _refused_sign_in( + identity[0], pending, upstream, org=identity[1], verified_email=email + ): return refused if pending["kind"] == "device": - return await device.signed_in(pending["user_code"], identity, upstream) + return await device.signed_in( + pending["user_code"], identity, upstream, verified_email=email + ) if pending["kind"] == "dashboard": - return await _dashboard_signed_in(identity, pending) - return await _connect_consent(pending, identity, upstream) + return await _dashboard_signed_in( + identity, pending, verified_email=email, upstream=upstream + ) + return await _connect_consent(pending, identity, upstream, verified_email=email) -async def _refused_if_deactivated( - email: str, pending: dict[str, Any], upstream: str +async def _refused_sign_in( + subject: str, pending: dict[str, Any], upstream: str, *, org: str = "", verified_email: str = "" ) -> Response | None: - """The answer to a sign-in by `email` that SCIM has deactivated (or that - cannot be checked), or None to carry on.""" + """A person refused by provisioning, a retryable read failure, or None to carry on.""" from server.auth import device, oidc from server.auth.login import SignInUnavailable try: - deactivated = await scim.deactivated(email) + deactivated = await scim.refuses_subject(subject, org=org, verified_email=verified_email) except SignInUnavailable: await oidc.revoke(upstream) return _sign_in_unavailable(_restart_url(pending)) if not deactivated: return None - await _signin_denied(email, _DEACTIVATED) + await _signin_denied(subject, _SIGN_IN_REFUSED) await oidc.revoke(upstream) if pending["kind"] == "device": await device.deny(pending["user_code"]) - return _denied_page(f"{_DEACTIVATED}; ask your Loom operator for access") + return _denied_page(f"{_SIGN_IN_REFUSED}; ask your Loom operator for access") def _link_cookie(ticket: str) -> str: @@ -950,7 +959,13 @@ async def link_account(request: Request) -> Response: upstream = unseal(UPSTREAM_PURPOSE, held["upstream_refresh"]) # Before any link or account is written: the page may have been open since before # a deactivation, and what it wrote would be reachable but never revoked. - if refused := await _refused_if_deactivated(email, sign_in, upstream): + if refused := await _refused_sign_in( + proven[0] if proven else email, + sign_in, + upstream, + org=proven[1] if proven else "", + verified_email=email, + ): refused.delete_cookie(_link_cookie(ticket), path="/oauth/link") return refused try: @@ -1047,10 +1062,15 @@ async def _connect_consent( pending: dict[str, Any], identity: Identity, upstream_refresh: str, + verified_email: str = "", ) -> Response: """Under OIDC the IdP may sign the user in without a single click, so the MCP client gets its code only after the user approves here, on a page that names the client and where the authorization goes.""" + if refused := await _refused_sign_in( + identity[0], pending, upstream_refresh, org=identity[1], verified_email=verified_email + ): + return refused client = await store.get_client(pending["client_id"]) if client is None: raise OAuthError("invalid_client") @@ -1065,6 +1085,7 @@ async def _connect_consent( "code_challenge": pending["code_challenge"], "mcp_state": pending["mcp_state"], "identity": list(identity), + "verified_email": scim.normalized(verified_email), "since": int(time.time()), # the identity was resolved now "upstream_refresh": seal(UPSTREAM_PURPOSE, upstream_refresh), }, @@ -1092,9 +1113,18 @@ async def _connect_consent( @connect_router.post("/oauth/authorize/approve") async def connect_approve(request: Request) -> Response: + from server.auth.login import SignInUnavailable + f = await form_body(request) try: return await _decide_connect(f) + except SignInUnavailable: + return html_page( + "Connect Loom", + "

Provisioning cannot be checked

Go back and submit your choice again.

", + 503, + headers={"Retry-After": "5"}, + ) except claims.Busy: # nothing was spent: the same choice can be sent again return html_page( "Connect Loom", @@ -1126,6 +1156,7 @@ async def _decide_connect(f: dict[str, str]) -> Response: upstream_refresh=unseal(UPSTREAM_PURPOSE, pending["upstream_refresh"]), state=pending["mcp_state"], since=pending["since"], + verified_email=pending.get("verified_email", ""), ) if await store.take_pending(ticket) is None: return expired # the other submit answered @@ -1144,11 +1175,17 @@ async def _decide_connect(f: dict[str, str]) -> Response: return _back_to_client(client["redirect_uri"], {"error": "access_denied"}, pending["mcp_state"]) -async def _dashboard_signed_in(identity: Identity, pending: dict[str, Any]) -> Response: +async def _dashboard_signed_in( + identity: Identity, pending: dict[str, Any], *, verified_email: str = "", upstream: str = "" +) -> Response: """Deliver a short-lived token at `/me`, which never redirects to another origin.""" from server.auth.revocation import subjects_revoked subject, org, namespaces, roles = identity + if refused := await _refused_sign_in( + subject, pending, upstream, org=org, verified_email=verified_email + ): + return refused issued_at = int(time.time()) # before the read: a revocation after it postdates the token try: revoked = await subjects_revoked([subject], issued_at=issued_at) @@ -1207,6 +1244,10 @@ async def issue_tokens( from server.auth.device import DEVICE_CLIENT_ID from server.auth.revocation import subjects_revoked + if await scim.refuses_subject( + grant["subject"], org=grant["org"], verified_email=grant.get("verified_email", "") + ): + raise _invalid_grant("this person cannot sign in under the cell's provisioning policy") issued_at = int(time.time()) # before the read: a revocation after it postdates the tokens if await subjects_revoked([grant["subject"]], issued_at=since): raise _invalid_grant("this account's tokens are revoked") @@ -1245,6 +1286,7 @@ async def issue_tokens( "family": grant["family"], "generation": generation, "upstream_refresh": grant["upstream_refresh"], + "verified_email": scim.normalized(grant.get("verified_email", "")), "held": grant.get("held", {}), # a rotation swaps from the token it spent }, ) @@ -1462,6 +1504,10 @@ async def _refresh_session(grant: Grant, refresh_token: str) -> dict[str, Any]: # ago on another process (or owed its revocation) mints nothing more. if await store.session_ended(grant["family"]): raise _invalid_grant() + if await scim.refuses_subject( + grant["subject"], org=grant["org"], verified_email=grant.get("verified_email", "") + ): + raise _invalid_grant("this person cannot sign in under the cell's provisioning policy") if settings.oauth_login_provider == "oidc": grant["upstream_refresh"] = await _ask_identity_provider(grant, refresh_token) elif grant["upstream_refresh"]: diff --git a/server/auth/oauth_store.py b/server/auth/oauth_store.py index 2b064588c2..3eef77b12d 100644 --- a/server/auth/oauth_store.py +++ b/server/auth/oauth_store.py @@ -52,6 +52,7 @@ class Grant(TypedDict): exp: int family: str upstream_refresh: str + verified_email: NotRequired[str] # A refresh token's place in its session's rotations (0 for the first). generation: NotRequired[int] # A code's: when its identity was resolved, the cutoff for revocations. @@ -74,6 +75,7 @@ class Device(TypedDict): exp: int family: str upstream_refresh: str + verified_email: NotRequired[str] user_code: str status: DeviceStatus address: str @@ -166,18 +168,33 @@ def _record(g: Grant | Device, fields: tuple[str, ...], spent: bool) -> dict[str """A grant as kept in Keeper: its IdP token sealed, and a nonce so no two versions are equal (claims.swap relies on it).""" record: dict[str, Any] = {f: g[f] for f in fields} # type: ignore[literal-required] + record["verified_email"] = g.get("verified_email", "") record["upstream_refresh"] = seal(UPSTREAM_PURPOSE, g["upstream_refresh"]) return record | {"spent": spent, "nonce": claims.nonce()} def _code(r: dict[str, Any]) -> Grant: g = {f: r[f] for f in _CODE_FIELDS} - return cast(Grant, g | {"upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"])}) + return cast( + Grant, + g + | { + "verified_email": r.get("verified_email", ""), + "upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"]), + }, + ) def _device(r: dict[str, Any]) -> Device: d = {f: r[f] for f in _DEVICE_FIELDS} - return cast(Device, d | {"upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"])}) + return cast( + Device, + d + | { + "verified_email": r.get("verified_email", ""), + "upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"]), + }, + ) # Not `refresh`, the type a build without sign-in sessions keeps its refresh @@ -197,6 +214,7 @@ def _refresh_row(rid: str, g: Grant, version: int) -> dict[str, Any]: "expires_at": int(g["exp"]), "family": g["family"], "generation": g.get("generation", 0), + "verified_email": g.get("verified_email", ""), "upstream_refresh": seal(UPSTREAM_PURPOSE, g["upstream_refresh"]), "version": version, } @@ -353,7 +371,7 @@ async def _refresh_grant(self, rid: str) -> Grant | None: with own_write_reads(): # handed out a moment ago, maybe through another replica row = await self._ch().query_one( f"""SELECT client_id, subject, org, namespaces, roles, expires_at, family, - generation, upstream_refresh + generation, upstream_refresh, verified_email FROM {_T} FINAL WHERE record_type = {{t:String}} AND id = {{id:String}} LIMIT 1""", params={"t": _REFRESH_TYPE, "id": rid}, @@ -371,6 +389,7 @@ async def _refresh_grant(self, rid: str) -> Grant | None: "exp": int(row["expires_at"]), "family": row["family"], "generation": int(row["generation"]), + "verified_email": row.get("verified_email", ""), "upstream_refresh": unseal(UPSTREAM_PURPOSE, row["upstream_refresh"]), } diff --git a/server/auth/routes.py b/server/auth/routes.py index f6ed9f113b..8c23ab17eb 100644 --- a/server/auth/routes.py +++ b/server/auth/routes.py @@ -13,7 +13,7 @@ from fastapi.responses import JSONResponse from server.auth.bearer import auth_unavailable, bearer_token -from server.auth.jwt import AuthUnavailable, TokenError, verify_token +from server.auth.jwt import AuthUnavailable, TokenError, authenticate_token from server.observability.metrics import record_auth_refusal from server.settings import settings @@ -105,7 +105,7 @@ async def whoami(request: Request) -> dict[str, object]: if token is None: return {"authenticated": False} try: - info = verify_token(token) + info = await authenticate_token(token) except TokenError as e: record_auth_refusal(e, transport="http") if not isinstance(e, AuthUnavailable): @@ -121,7 +121,7 @@ async def whoami(request: Request) -> dict[str, object]: except pyjwt.PyJWTError: header = {} - # exp/iat surfaced without re-verifying (verify_token already passed) + # exp/iat surfaced without re-verifying (authenticate_token already passed) try: unverified_claims = pyjwt.decode( token, diff --git a/server/auth/scim.py b/server/auth/scim.py index c99db5f447..11f8e0b9d6 100644 --- a/server/auth/scim.py +++ b/server/auth/scim.py @@ -24,9 +24,11 @@ from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack, asynccontextmanager from datetime import UTC, datetime, timedelta -from typing import Any +from typing import TYPE_CHECKING, Any from server.auth import login, revocation, service_accounts +from server.auth.scim_people import refuses_subject as refuses_subject +from server.auth.scim_people import requires_provisioning as requires_provisioning from server.core import leases, scim_tokens from server.core.ch import get_client, own_write_reads from server.core.datetimes import as_aware_utc @@ -36,6 +38,32 @@ from server.observability import audit from server.settings import settings +if TYPE_CHECKING: + from server.auth.jwt import AuthInfo + + +async def refuses_auth(auth: AuthInfo, *, org: str | None = None) -> bool: + """Check the effective person and every bearer, creator and parent behind it.""" + from server.auth.gateway import is_end_user + from server.auth.scim_people import MAX_ITEMS + + if ( + is_end_user(auth) + and auth.impersonated_by + and auth.read_scope == "own_user" + and auth.read_user_id + ): + return False + if len(auth.parent_subjects) >= MAX_ITEMS: + return True + extra = (*auth.parent_subjects,) + if auth.creator: + extra += (auth.creator,) + if auth.impersonated_by: + extra += (auth.impersonated_by,) + return await refuses_subject(auth.subject, org=auth.org if org is None else org, also=extra) + + log = logging.getLogger(__name__) TABLE = "loom.scim_users" @@ -429,6 +457,7 @@ async def _write_audited(row: dict[str, Any], new: dict[str, Any]) -> None: audit nothing: so the row is read back, and what landed is audited.""" from server.auth.authz import forget_ended + forget_ended("scim", row["email"]) try: if new != row or "version" not in row: # a change, or a user not yet written await _write(new) diff --git a/server/auth/scim_people.py b/server/auth/scim_people.py new file mode 100644 index 0000000000..033b198c49 --- /dev/null +++ b/server/auth/scim_people.py @@ -0,0 +1,306 @@ +"""Bounded active-person decisions for a cell's complete admission ancestry.""" + +from __future__ import annotations + +import asyncio +import json +import math +import time +from dataclasses import dataclass, field +from typing import Any + +from server.core import cell, ch +from server.core.datetimes import as_aware_utc +from server.core.grant_sql import VERIFIED_EMAIL_SQL +from server.settings import settings + +MAX_ITEMS = 1024 # not a setting: security work per complete decision +MAX_BYTES = 65536 # not a setting: input, parameters and transferred identity data +SCIM_DECISION_S = 30.0 # not a setting: even an unbounded generic read must finish here + + +class _Refused(Exception): + """The complete identity graph cannot qualify within its supported size.""" + + +def _now() -> float: + return time.monotonic() + + +def _encoded(value: Any) -> int: + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()) + + +def _roots(subject: str, verified_email: str, also: tuple[str, ...], org: str) -> tuple[str, ...]: + if not isinstance(also, (list, tuple)) or len(also) >= MAX_ITEMS: + raise _Refused + size = 0 + for value in (subject, verified_email, org, *also): + if not isinstance(value, str) or len(value) > MAX_BYTES or "\x00" in value: + raise _Refused + size += len(value.encode()) + if size > MAX_BYTES: + raise _Refused + if not subject.strip() or any(not value.strip() for value in also): + raise _Refused + if _encoded((subject, verified_email, also, org)) > MAX_BYTES: + raise _Refused + return tuple(sorted(set(also) - {subject})) + + +def _decision_seconds() -> float: + configured = float(settings.ch_timeout_read_s) + return ( + min(SCIM_DECISION_S, configured) + if math.isfinite(configured) and configured > 0 + else SCIM_DECISION_S + ) + + +def requires_provisioning() -> bool: + return bool(settings.cell and settings.scim_token and cell.mode() == "enforced") + + +async def refuses_subject( + subject: str, *, org: str, verified_email: str = "", also: tuple[str, ...] = () +) -> bool: + from server.auth import authz, login, scim + + try: + extra = _roots(subject, verified_email, also, org) + except (UnicodeError, _Refused): + return True + email = scim.normalized(verified_email) + required = requires_provisioning() + if not required: + for address in {scim.normalized(s) for s in (subject, email, *extra) if "@" in s}: + if await authz._remembered(("scim", address), scim.deactivated): + return True + return False + key = ("scim_people", cell.deployment_id(), settings.cell, org, email, extra, subject) + + async def read(primary: str) -> bool: + seconds = _decision_seconds() + decision = _Decision(org, primary, email, _now() + seconds) + try: + async with asyncio.timeout(seconds): + with ch.own_write_reads(), ch.query_tag("auth.scim_people"): + return await decision.refuses((primary, *extra)) + except _Refused: + return True + except Exception as e: # an unchecked identity never admits a person + raise login.SignInUnavailable from e + + return await authz._remembered(key, read) + + +@dataclass +class _Decision: + org: str + primary: str + verified_email: str + deadline: float + visiting: set[str] = field(default_factory=set) + completed: set[str] = field(default_factory=set) + emails: set[str] = field(default_factory=set) + edges: int = 0 + accounts: int = 0 + transferred: int = 0 + + def remaining(self) -> float: + seconds = self.deadline - _now() + if seconds <= 0: + raise TimeoutError + configured = ch.class_timeout_s("read") + return min(seconds, configured) if math.isfinite(configured) and configured > 0 else seconds + + def transferred_bytes(self, count: int) -> None: + self.transferred += count + if self.transferred > MAX_BYTES: + raise _Refused + + async def query(self, sql: str, params: dict[str, Any], *, many: bool = False) -> Any: + if _encoded(params) > MAX_BYTES: + raise _Refused + with ch.statement_timeout(self.remaining()): + client = ch.get_client() + if many: + return await client.query_all(sql, params=params) + return await client.query_one(sql, params=params) + + async def query_one(self, sql: str, params: dict[str, Any]) -> Any: + """Keep the shared ancestry reader within this decision's query bounds.""" + return await self.query(sql, params) + + async def refuses(self, roots: tuple[str, ...]) -> bool: + for root in roots: + stack = [(root, False)] + while stack: + self.remaining() + subject, leaving = stack.pop() + if leaving: + self.visiting.remove(subject) + self.completed.add(subject) + continue + if subject in self.visiting: + return True + if subject in self.completed: + continue + if len(self.visiting) + len(self.completed) >= MAX_ITEMS: + raise _Refused + self.visiting.add(subject) + stack.append((subject, True)) + if subject.lower().startswith("sa:"): + ancestors = await self.ancestors(subject) + self.edges += len(ancestors) + if self.edges > MAX_ITEMS: + raise _Refused + stack.extend((ancestor, False) for ancestor in reversed(ancestors)) + elif await self.person_refused(subject): + return True + return False + + async def ancestors(self, subject: str) -> list[str]: + from server.auth import service_accounts + + name = subject.lower().removeprefix("sa:").partition(":")[0] + if not name: + raise _Refused + bot = await self.query( + "SELECT active FROM loom.machine_identities FINAL WHERE name = {name:String} LIMIT 1", + {"name": name}, + ) + if bot is not None: + if not bot["active"]: + raise _Refused + return [] + row = await service_accounts._creator_row( + self.org, + subject, + ch=self, + max_items=MAX_ITEMS - self.edges, + max_bytes=MAX_BYTES - self.transferred, + ) + if row is None or row["revoked_at"] is not None: + raise _Refused + expires_at = row.get("expires_at") + if expires_at is not None and as_aware_utc(expires_at).timestamp() <= time.time(): + raise _Refused + if int(row["lineage_count"]) > MAX_ITEMS - self.edges: + raise _Refused + self.transferred_bytes(int(row["subject_bytes"])) + creator = row["created_by"] + lineage = row["lineage"] + if any(not isinstance(s, str) or not s.strip() or "\x00" in s for s in (creator, *lineage)): + raise _Refused + return [creator, *lineage] + + async def account_emails(self, subject: str) -> tuple[set[str], set[str]]: + from server.auth import scim + + folded = scim.normalized(subject) + mandatory = {folded} if "@" in folded else set() + witnesses = set(mandatory) + if subject == self.primary and self.verified_email: + mandatory.add(self.verified_email) + witnesses.add(self.verified_email) + if self.org not in ("", "*", "-"): + owner = await self.owner(folded) + if owner is not None: + if "@" in owner: + mandatory.add(owner) + witnesses.add(owner) + await self.contacts(owner, mandatory, witnesses) + self.emails.update(mandatory | witnesses) + if len(self.emails) > MAX_ITEMS: + raise _Refused + return mandatory, witnesses + + async def owner(self, subject: str) -> str | None: + rows = await self.query( + f"""SELECT DISTINCT + if(length(lower(owner_email)) <= {{bytes:UInt32}}, lower(owner_email), '') AS owner, + length(lower(owner_email)) > {{bytes:UInt32}} AS oversized + FROM loom.namespaces FINAL + WHERE org = {{org:String}} AND ifNull(active, true) = true + AND (lower(owner_email) = {{sub:String}} + OR (lower(contact_email) = {{sub:String}} AND {VERIFIED_EMAIL_SQL})) + LIMIT 2""", + {"org": self.org, "sub": subject, "bytes": MAX_BYTES - self.transferred}, + many=True, + ) + if len(rows) > 1 or any(r["oversized"] for r in rows): + raise _Refused + if not rows: + return None + owner = str(rows[0]["owner"]) + if not owner: + raise _Refused + self.transferred_bytes(len(owner.encode())) + return owner + + async def contacts(self, owner: str, mandatory: set[str], witnesses: set[str]) -> None: + from server.auth import scim + + row = await self.query( + f"""SELECT count() AS account_count, sum(length(contact_email)) AS account_bytes, + if(account_count <= {{items:UInt32}} AND account_bytes <= {{bytes:UInt32}}, + groupArray({MAX_ITEMS + 1})(( + if(length(contact_email) <= {{bytes:UInt32}}, lower(contact_email), ''), + {VERIFIED_EMAIL_SQL})), []) AS accounts + FROM loom.namespaces FINAL + WHERE org = {{org:String}} AND lower(owner_email) = {{owner:String}} + AND ifNull(active, true) = true + LIMIT 1""", + { + "org": self.org, + "owner": owner, + "items": MAX_ITEMS - self.accounts, + "bytes": MAX_BYTES - self.transferred, + }, + ) + self.accounts += int(row["account_count"]) + if self.accounts > MAX_ITEMS: + raise _Refused + self.transferred_bytes(int(row["account_bytes"])) + for email, verified in row["accounts"]: + email = scim.normalized(email) + if email: + mandatory.add(email) + if verified: + witnesses.add(email) + + async def person_refused(self, subject: str) -> bool: + from server.auth import scim + + mandatory, witnesses = await self.account_emails(subject) + if not witnesses: + return True + by_id = {scim.user_id(email): email for email in mandatory} + rows = await self.query( + """SELECT id, active, deleted FROM loom.scim_users FINAL + WHERE id IN {ids:Array(String)} LIMIT 1024""", + {"ids": list(by_id)}, + many=True, + ) + found = set() + for row in rows: + if not row["active"] or row["deleted"]: + return True + found.add(by_id[row["id"]]) + missing = mandatory - found + if not missing: + return False + row = await self.query( + """SELECT groupUniqArray(1025)(address) AS qualified FROM ( + SELECT arrayJoin(arrayIntersect(work_addresses, {missing:Array(String)})) AS address + FROM loom.scim_users FINAL WHERE id IN {ids:Array(String)} + AND active = 1 AND deleted = 0 AND email IN {witnesses:Array(String)} + ) LIMIT 1""", + { + "ids": [scim.user_id(email) for email in witnesses], + "witnesses": sorted(witnesses), + "missing": sorted(missing), + }, + ) + return not missing.issubset(row["qualified"]) diff --git a/server/auth/self_signup.py b/server/auth/self_signup.py index 8e33918611..717bd86e67 100644 --- a/server/auth/self_signup.py +++ b/server/auth/self_signup.py @@ -71,31 +71,34 @@ async def provision_account( can't drift. password_hash="" = no password sign-in. `ch` defaults to the server's client. """ + from server.auth.authz import identity_write + ch = ch or get_client() now = naive_utc_now_seconds() - await ch.insert( - "loom.namespaces", - [ - { - "org": org, - "namespace": namespace, - "owner_email": subject, - "contact_email": contact_email, - "password_hash": password_hash, - "description": description, - "tags": list(tags or []), - "active": True, - "conflict_threshold": None, - "retention_days": 0, - "warm_days": 0, - "visibility": "private", - "classification": "standard", - "join_policy": "inherit", - "created_at": now, - "updated_at": now, - } - ], - ) + with identity_write(): + await ch.insert( + "loom.namespaces", + [ + { + "org": org, + "namespace": namespace, + "owner_email": subject, + "contact_email": contact_email, + "password_hash": password_hash, + "description": description, + "tags": list(tags or []), + "active": True, + "conflict_threshold": None, + "retention_days": 0, + "warm_days": 0, + "visibility": "private", + "classification": "standard", + "join_policy": "inherit", + "created_at": now, + "updated_at": now, + } + ], + ) await insert_owner_grant(org, namespace, subject, ch=ch) @@ -597,6 +600,8 @@ async def _single_user_agent(input: AgentSignupInput, request: Request) -> Agent async def _provision_agent( input: AgentSignupInput, request: Request, org: str, namespace: str ) -> AgentSignupOutput: + from server.auth.authz import identity_write + ch = get_client() existing = await ch.query_one( """ @@ -653,29 +658,30 @@ async def _provision_agent( settings={"async_insert": 0, "wait_for_async_insert": 1}, ) await grant_cache.changed(org, [sa_subject]) - await ch.insert( - "loom.service_accounts", - [ - { - "org": org, - "name": f"{AGENT_NAME_PREFIX}{input.name.strip()}", - "sa_subject": sa_subject, - "namespaces": [namespace], - "roles": ["writer"], - "ttl_days": int(input.ttl_days or 0), - "created_by": namespace, - "created_at": now, - "expires_at": expires_at, - "revoked_at": None, - "revoked_by": "", - "version": time.time_ns(), - "kind": "agent", - "client": AGENT_CLIENT, - "inherit_creator_grants": False, - "ns_roles": json.dumps({namespace: "writer"}), - } - ], - ) + with identity_write(): + await ch.insert( + "loom.service_accounts", + [ + { + "org": org, + "name": f"{AGENT_NAME_PREFIX}{input.name.strip()}", + "sa_subject": sa_subject, + "namespaces": [namespace], + "roles": ["writer"], + "ttl_days": int(input.ttl_days or 0), + "created_by": namespace, + "created_at": now, + "expires_at": expires_at, + "revoked_at": None, + "revoked_by": "", + "version": time.time_ns(), + "kind": "agent", + "client": AGENT_CLIENT, + "inherit_creator_grants": False, + "ns_roles": json.dumps({namespace: "writer"}), + } + ], + ) return AgentSignupOutput( token=token, namespace=namespace, diff --git a/server/auth/service_accounts.py b/server/auth/service_accounts.py index ed5e34de11..0d975a878a 100644 --- a/server/auth/service_accounts.py +++ b/server/auth/service_accounts.py @@ -76,25 +76,14 @@ async def sa_creator(org: str, sa_subject: str, *, ch: Any | None = None) -> SAC hit = _creators.get(key) if hit is not None and time.monotonic() - hit[0] < settings.jwt_revocation_cache_ttl_seconds: return hit[1] - if ch is None: - from server.core.ch import get_client - - ch = get_client() try: - row = await ch.query_one( - """ - SELECT created_by, inherit_creator_grants, kind - FROM loom.service_accounts FINAL - WHERE org = {org:String} AND sa_subject = {sub:String} - AND revoked_at IS NULL - LIMIT 1 - """, - params={"org": org, "sub": sa_subject}, - ) + row = await _creator_row(org, sa_subject, ch=ch) except Exception as e: # noqa: BLE001 — a failed read inherits nothing (fail closed) log.warning("service_accounts lookup failed for %s/%s: %s", org, sa_subject, e) note_unreadable() return None + if row is not None and row["revoked_at"] is not None: + return None creator = str((row or {}).get("created_by") or "") if not creator: return None @@ -109,6 +98,36 @@ async def sa_creator(org: str, sa_subject: str, *, ch: Any | None = None) -> SAC return found +async def _creator_row( + org: str, + sa_subject: str, + *, + ch: Any | None = None, + max_items: int = 1024, + max_bytes: int = 65536, +) -> dict[str, Any] | None: + """One current row, with ancestry projected only within the caller's budget.""" + if ch is None: + from server.core.ch import get_client + + ch = get_client() + row = await ch.query_one( + """SELECT inherit_creator_grants, kind, revoked_at, expires_at, + length(lineage) AS lineage_count, + arraySum(arrayMap(s -> length(s), lineage)) + length(created_by) AS subject_bytes, + if(length(created_by) <= {bytes:UInt32}, created_by, '') AS bounded_creator, + if(lineage_count <= {items:UInt32} AND subject_bytes <= {bytes:UInt32}, + lineage, []) AS bounded_lineage + FROM loom.service_accounts FINAL + WHERE org = {org:String} AND sa_subject = {sub:String} LIMIT 1""", + params={"org": org, "sub": sa_subject, "items": max_items, "bytes": max_bytes}, + ) + if row is not None: + row["created_by"] = row.pop("bounded_creator") + row["lineage"] = row.pop("bounded_lineage") + return row + + async def inheriting_creator(org: str, sa_subject: str, *, ch: Any | None = None) -> str | None: """The creator whose live grants `sa_subject` reads through, or None: the account does not inherit, has no creator, or is its own creator.""" @@ -207,6 +226,7 @@ async def add(child: str) -> None: async def _add_lineage( org: str, name: str, find: str, params: dict[str, Any], above: list[str] ) -> None: + from server.auth.authz import identity_write from server.auth.grant_claims import service_account_change from server.core.ch import get_client, own_write_reads @@ -222,7 +242,10 @@ async def _add_lineage( lineage = sorted({*row["lineage"], *above}) # Every column carried, past the version read, as `revoked_row` does. version = max(time.time_ns(), int(row["version"]) + 1) - await ch.insert("loom.service_accounts", [{**row, "lineage": lineage, "version": version}]) + with identity_write(): + await ch.insert( + "loom.service_accounts", [{**row, "lineage": lineage, "version": version}] + ) # Names a pass revokes at once: each is a lease round trip and an insert. @@ -328,6 +351,8 @@ async def _revoke_all( listed = {str(r["sa_subject"]) for r in found} async def revoke(org: str, name: str) -> str | None: + from server.auth.authz import identity_write + async with gate, service_account_change(org, name): with own_write_reads(): row = await ch.query_one( @@ -348,7 +373,8 @@ async def revoke(org: str, name: str) -> str | None: revoked_copy = revoked_row( row, revoked_at=naive_utc_now_seconds(), revoked_by=revoked_by ) - await ch.insert("loom.service_accounts", [revoked_copy]) + with identity_write(): + await ch.insert("loom.service_accounts", [revoked_copy]) return str(row["sa_subject"]) try: diff --git a/server/auth/sso_links.py b/server/auth/sso_links.py index 8ff3467b74..13d4d359b4 100644 --- a/server/auth/sso_links.py +++ b/server/auth/sso_links.py @@ -149,6 +149,8 @@ async def _write( ) -> None: """Write the link rows through `ch`; the grant watermark goes through the default client (`grant_cache.changed`), as every grant write's does.""" + from server.auth.authz import identity_write + if not accounts: return ch = ch or get_client() @@ -164,21 +166,22 @@ async def _write( for org in orgs: await announced.enter_async_context(grant_cache.announced(org)) try: - await ch.insert( - TABLE, - [ - { - "email": email.strip().lower(), - "org": org, - "namespace": namespace, - "linked": linked, - "linked_by": by, - "linked_at": now, - "version": version, - } - for org, namespace in accounts - ], - ) + with identity_write(): + await ch.insert( + TABLE, + [ + { + "email": email.strip().lower(), + "org": org, + "namespace": namespace, + "linked": linked, + "linked_by": by, + "linked_at": now, + "version": version, + } + for org, namespace in accounts + ], + ) finally: for org in orgs: aliases.invalidate_org(org) diff --git a/server/cli/hygiene.py b/server/cli/hygiene.py index e60e34feb1..589228211d 100644 --- a/server/cli/hygiene.py +++ b/server/cli/hygiene.py @@ -312,6 +312,7 @@ async def _adopt_run(ch: Any, args: argparse.Namespace) -> int: from datetime import UTC, datetime import server.operations # noqa: F401 — wires handlers before importing one + from server.auth.authz import identity_write from server.identity.api.models import NamespaceRegisterInput from server.identity.app.namespace import registry_row from server.identity.app.namespace_autoregister import ( @@ -370,7 +371,8 @@ def _kind(name: str) -> str: for row in batch: row["updated_at"] = AUTO_ROW_VERSION # an explicit row always wins for i in range(0, len(batch), 500): - await ch.insert("loom.namespaces", batch[i : i + 500]) + with identity_write(): + await ch.insert("loom.namespaces", batch[i : i + 500]) print(f"adopted {len(batch)} namespace(s); retention_days=0 on every row, no TTL altered") return 0 diff --git a/server/code/app/index/registry.py b/server/code/app/index/registry.py index 9f5fe377c9..3169acff00 100644 --- a/server/code/app/index/registry.py +++ b/server/code/app/index/registry.py @@ -89,6 +89,8 @@ async def ensure_registered( async def insert_first_row(cfg: Any, client: Any, *, restricted: bool, join_policy: str) -> None: """Write a code namespace's first directory row. The caller holds the namespace's account and admins claims, under which it found no row.""" + from server.auth.authz import identity_write + now = datetime.now(UTC).replace(microsecond=0, tzinfo=None) row = { "org": cfg.org, @@ -108,7 +110,8 @@ async def insert_first_row(cfg: Any, client: Any, *, restricted: bool, join_poli "created_at": now, "updated_at": now, } - await client.insert("loom.namespaces", [row]) + with identity_write(): + await client.insert("loom.namespaces", [row]) async def _access_row(cfg: Any, client: Any) -> dict[str, Any] | None: @@ -145,6 +148,7 @@ async def _register_first_row(cfg: Any, client: Any, *, restricted: bool, public async def _write_row(cfg: Any, client: Any) -> bool: """`ensure_registered`'s directory write; True when a row was inserted.""" + from server.auth.authz import identity_write from server.identity.app.namespace_classification import note_restricted from server.identity.app.registration import namespace_row @@ -186,7 +190,8 @@ async def _write_row(cfg: Any, client: Any) -> bool: # readable — the row stays visibility=private; reads need the join. opened = dict(row) opened.update(join_policy="open", visibility="private", updated_at=now) - await client.insert("loom.namespaces", [opened]) + with identity_write(): + await client.insert("loom.namespaces", [opened]) logger.info( "code namespace %s/%s directory row set open (%s says public)", cfg.org, @@ -198,7 +203,8 @@ async def _write_row(cfg: Any, client: Any) -> bool: narrowed["classification"] = "restricted" narrowed["visibility"] = "private" narrowed["updated_at"] = now - await client.insert("loom.namespaces", [narrowed]) + with identity_write(): + await client.insert("loom.namespaces", [narrowed]) note_restricted(cfg.org, cfg.namespace) logger.warning( "code namespace %s/%s directory row narrowed to restricted " @@ -214,7 +220,8 @@ async def _write_row(cfg: Any, client: Any) -> bool: widened["classification"] = "standard" widened["join_policy"] = str((access or {}).get("join_policy") or "approval") widened["updated_at"] = now - await client.insert("loom.namespaces", [widened]) + with identity_write(): + await client.insert("loom.namespaces", [widened]) logger.warning( "code namespace %s/%s directory row widened to standard (a maintainer's " "access row says so; the row said restricted)", diff --git a/server/code/app/repo_admin.py b/server/code/app/repo_admin.py index af02d226df..d5d06fc3be 100644 --- a/server/code/app/repo_admin.py +++ b/server/code/app/repo_admin.py @@ -715,6 +715,8 @@ async def handle_code_repo_set_access(input: Any) -> Any: needs `confirm` = namespace; a restricted index is never `open`. The change then syncs the repository's maintainers, quietly: a restriction's failed sync is finished by the ingest pass, a widening's by the next sync of the repository.""" + from server.auth.authz import identity_write + require_code_intel() from server.code.domain.index.repos import repo_for_namespace from server.core.ch import get_client @@ -787,7 +789,8 @@ async def handle_code_repo_set_access(input: Any) -> Any: "updated_at": datetime.now(UTC).replace(microsecond=0, tzinfo=None), } ) - await ch.insert("loom.namespaces", [new_row]) + with identity_write(): + await ch.insert("loom.namespaces", [new_row]) directory_updated = True await reg.upsert_access( cfg, diff --git a/server/erasure/app/saga_steps.py b/server/erasure/app/saga_steps.py index 848ea5e267..c44c71b8a5 100644 --- a/server/erasure/app/saga_steps.py +++ b/server/erasure/app/saga_steps.py @@ -182,8 +182,11 @@ async def _step_remove_from_groups_by_user(org: str, user_id: str) -> int: async def _step_retire_personal_collection_by_user(org: str, user_id: str) -> int: """Under its directory row's claim, which `namespace.delete` holds too.""" + from server.auth.authz import identity_write + async with account_change(org, personal_name(org, "user:" + user_id)): - return await principal_steps.retire_personal_collection_by_user(org, user_id) + with identity_write(): + return await principal_steps.retire_personal_collection_by_user(org, user_id) async def _step_retire_knowledge_sources_by_ns(org: str, namespace: _NS) -> int: @@ -244,12 +247,15 @@ async def _step_forget_mirrors_by_ns(org: str, namespace: _NS) -> int: async def _step_forget_collection_owner_by_user(org: str, user_id: str) -> int: """Under each directory row's claim, then the tree's, so neither a `namespace.delete` nor a move that lands meanwhile writes back what it read.""" + from server.auth.authz import identity_write + async with AsyncExitStack() as held: for namespace in await principal_steps.collections_created_by(org, user_id): await held.enter_async_context(account_change(org, namespace)) await held.enter_async_context(collections_change(org)) - # No watermark: the cached tree holds parents, not owners. - return await principal_steps.forget_collection_owner_by_user(org, user_id) + # The tree cache holds parents; the person memo also reads these owners. + with identity_write(): + return await principal_steps.forget_collection_owner_by_user(org, user_id) async def _step_tombstone_documents_by_user(org: str, user_id: str) -> int: diff --git a/server/health.py b/server/health.py index 3cae7c8b2d..f1f2aba911 100644 --- a/server/health.py +++ b/server/health.py @@ -48,19 +48,19 @@ async def _otel_unindexed_for_health(ch_ok: bool) -> tuple[str, ...] | None: return None -def _is_platform_admin(request: Request) -> bool: +async def _is_platform_admin(request: Request) -> bool: """Whether the caller's bearer token verifies as a platform admin's, who reads each code-index entry that is not restricted. No token, or one that does not verify, is an anonymous caller: `/v1/health` refuses no one.""" from server.auth.bearer import bearer_token - from server.auth.jwt import TokenError, verify_token + from server.auth.jwt import TokenError, authenticate_token from server.operations.base import Role try: token = bearer_token(request.headers.get("Authorization")) if token is None: return False - info = verify_token(token) + info = await authenticate_token(token) except TokenError: return False # the admin routes' rule: a scoped token's `ns_roles` is no deployment authority @@ -394,7 +394,7 @@ async def health(request: Request, response: Response, include: str = "") -> dic requested = {s.strip() for s in include.split(",") if s.strip()} ch_ok, ch_error = await _ping_clickhouse() - platform_admin = _is_platform_admin(request) + platform_admin = await _is_platform_admin(request) prov = _health_provider_fields( ch_ok, await _otel_unindexed_for_health(ch_ok), platform_admin=platform_admin ) diff --git a/server/identity/app/account_admin.py b/server/identity/app/account_admin.py index 242ca60aae..0e9d0d2f51 100644 --- a/server/identity/app/account_admin.py +++ b/server/identity/app/account_admin.py @@ -262,6 +262,7 @@ async def link_account( SSO sign-in makes a second account for it meanwhile. Returns the address it had, the namespaces carried, and whether that address is another account's verified email, as read under the account and email claims.""" + from server.auth.authz import identity_write from server.auth.self_signup import CLAIM_TTL_S, email_claim from server.core import leases from server.core.ch import get_client @@ -283,10 +284,11 @@ async def link_account( if was: # first, so a link stopped part way leaves no link a re-run can't redo await sso_links.unlink(was, [(org, namespace)], by) # Insert-as-replace carrying every other column forward. - await get_client().insert( - "loom.namespaces", - [{**row, "contact_email": email, "updated_at": max(now, row["updated_at"])}], - ) + with identity_write(): + await get_client().insert( + "loom.namespaces", + [{**row, "contact_email": email, "updated_at": max(now, row["updated_at"])}], + ) await sso_links.link(email, [(org, namespace)], by) await audit.emit( "auth.sso_linked", diff --git a/server/identity/app/agent_claim.py b/server/identity/app/agent_claim.py index cbc8509769..bced8ea66d 100644 --- a/server/identity/app/agent_claim.py +++ b/server/identity/app/agent_claim.py @@ -378,14 +378,17 @@ async def _landed(org: str, namespace: str, password_hash: str) -> bool: async def _write(row: dict[str, Any], password_hash: str) -> None: """Replace the code with the claimer's password and mark the account claimed.""" + from server.auth.authz import identity_write + now = datetime.now(UTC).replace(tzinfo=None, microsecond=0) description = str(row["description"]) if description == AGENT_DESCRIPTION_UNCLAIMED: description = AGENT_DESCRIPTION - await get_client().insert( - "loom.namespaces", - [namespaces.claimed_row(row, password_hash, CLAIMED_TAG, description, now)], - ) + with identity_write(): + await get_client().insert( + "loom.namespaces", + [namespaces.claimed_row(row, password_hash, CLAIMED_TAG, description, now)], + ) async def _audit(org: str, namespace: str, *, confirmed: bool = True) -> None: diff --git a/server/identity/app/claim_release.py b/server/identity/app/claim_release.py index 1a98748e23..6eca449311 100644 --- a/server/identity/app/claim_release.py +++ b/server/identity/app/claim_release.py @@ -155,6 +155,7 @@ async def _release( # SELECT * so the release row mirrors EVERY column forward: loom.namespaces is # ReplacingMergeTree(updated_at), and an omitted column would reset to its schema # default under FINAL (a restricted namespace would read back as standard). + from server.auth.authz import identity_write from server.identity.store import namespaces as namespaces_repo existing = await namespaces_repo.fetch_row(org, namespace, ch=ch) @@ -221,7 +222,8 @@ async def _release( row["description"] = f"Released by {actor_subject}" + (f" ({reason})" if reason else "") row["updated_at"] = max(now, existing.get("updated_at") or now) rewritten.append(True) - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) # A bot's exchange reads its live namespaces sequentially, so no token # issued from here on carries this one: the bots' cutoff is taken now. diff --git a/server/identity/app/collections.py b/server/identity/app/collections.py index 312920b143..e4348a53cc 100644 --- a/server/identity/app/collections.py +++ b/server/identity/app/collections.py @@ -113,6 +113,8 @@ async def _write_new(input: CollectionCreateInput, namespace: str, caller: str) """The tree row and the creator's grant first, the directory row last: a create cut short is finished by its creator's retry, and the name is never left with no admin.""" + from server.auth.authz import identity_write + org, parent = input.org, input.parent if parent: check_placement(await sharing.load_tree(org), namespace, parent) @@ -138,13 +140,16 @@ async def _write_new(input: CollectionCreateInput, namespace: str, caller: str) ) row["kind"] = "collection" row["title"] = input.title - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) async def _give_title(org: str, namespace: str, title: str) -> str: """The collection's title: its own, else `title`, stored on it then when the caller is its admin (it was made with none, by a build without titles or by name alone). The caller holds the name's account claim.""" + from server.auth.authz import identity_write + with own_write_reads(): row = await namespaces_repo.fetch_row(org, namespace) if row is None or row.get("title") or not title: @@ -154,7 +159,8 @@ async def _give_title(org: str, namespace: str, title: str) -> str: # Never behind the row it replaces (on a tie the row inserted last wins); a delete a # moment later stamps the same way, so it still wins. stamp = max(naive_utc_now_seconds(), row["updated_at"]) - await get_client().insert("loom.namespaces", [{**row, "title": title, "updated_at": stamp}]) + with identity_write(): + await get_client().insert("loom.namespaces", [{**row, "title": title, "updated_at": stamp}]) return title diff --git a/server/identity/app/conversation_logging.py b/server/identity/app/conversation_logging.py index 96ef618e89..114c86dae6 100644 --- a/server/identity/app/conversation_logging.py +++ b/server/identity/app/conversation_logging.py @@ -14,6 +14,7 @@ async def set_conversation_logging(org: str, namespace: str, enabled: bool) -> N written to but never registered has no row, so one is created: a mutation there would match nothing and the Record conversations switch would report On for a namespace that never records.""" + from server.auth.authz import identity_write from server.core.ch import get_client ch = get_client() @@ -37,4 +38,5 @@ async def set_conversation_logging(org: str, namespace: str, enabled: bool) -> N ) row["conversation_logging_enabled"] = enabled row["updated_at"] = now - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) diff --git a/server/identity/app/me_callers.py b/server/identity/app/me_callers.py index 868f8bd1b1..11450e3985 100644 --- a/server/identity/app/me_callers.py +++ b/server/identity/app/me_callers.py @@ -81,27 +81,27 @@ async def caller_role_on(org: str, namespace: str, subject: str) -> str | None: return r if grant_rank(r) else None -def require_authed_caller(request: Request) -> tuple[str, str]: +async def require_authed_caller(request: Request) -> tuple[str, str]: """Return (subject, org) for the authenticated caller. Raises 401 if no valid bearer. In dev mode (auth_required=false, no bearer), falls back to a body-supplied identity via the calling endpoint.""" - info = _authed(request) + info = await _authed(request) remember_caller(request, info.subject, info.org) return info.subject, info.org -def require_authed_info(request: Request) -> AuthInfo: +async def require_authed_info(request: Request) -> AuthInfo: """`require_authed_caller`, returning the verified token itself.""" - info = _authed(request) + info = await _authed(request) remember_caller(request, info.subject, info.org) return info -def _authed(request: Request) -> AuthInfo: +async def _authed(request: Request) -> AuthInfo: """The verified bearer's principal; 401 when the request carries none.""" from server.auth.bearer import missing_bearer, verify_bearer_header - info = verify_bearer_header(request.headers.get("Authorization")) + info = await verify_bearer_header(request.headers.get("Authorization")) if info is None: raise missing_bearer() return info @@ -134,36 +134,18 @@ async def lowered_aliases(org: str, subject: str) -> list[str]: return lowered(await subject_aliases_of(org, subject)) -def caller_identity(request: Request, *, prefer_bearer_in_dev: bool = False) -> tuple[str, str]: - """(subject, org) for a caller-API caller, tolerant of dev mode. - - Auth-on: verify the bearer (401 if missing/invalid). Auth-off: a synthetic - single-user identity, unless a self-minted bearer is present and - `prefer_bearer_in_dev` (so per-subject role lookups stay accurate on - multi-member namespaces). A wildcard org falls back to self_signup_org. - """ - from server.auth.bearer import bearer_token, gateway_refused - from server.auth.gateway import is_end_user - from server.auth.jwt import TokenError, verify_token - - if settings.auth_actually_required(): - info = _authed(request) - org = info.org if info.org and info.org != "*" else settings.self_signup_org - remember_caller(request, info.subject, org) - return info.subject, org - if prefer_bearer_in_dev: - try: - token = bearer_token(request.headers.get("Authorization")) - if token is not None: - info = verify_token(token) - if is_end_user(info): - raise gateway_refused() - org = info.org if info.org and info.org != "*" else settings.self_signup_org - remember_caller(request, info.subject, org) - return info.subject, org - except TokenError: - pass - return "you", settings.self_signup_org +async def caller_identity(request: Request) -> tuple[str, str]: + """The supplied bearer's identity, or the deliberate no-bearer dev identity.""" + from server.auth.bearer import missing_bearer, verify_bearer_header + + info = await verify_bearer_header(request.headers.get("Authorization")) + if info is None: + if settings.auth_actually_required(): + raise missing_bearer() + return "you", settings.self_signup_org + org = info.org if info.org and info.org != "*" else settings.self_signup_org + remember_caller(request, info.subject, org) + return info.subject, org __all__ = [ diff --git a/server/identity/app/me_routes.py b/server/identity/app/me_routes.py index 0b1f70432a..b6470b3478 100644 --- a/server/identity/app/me_routes.py +++ b/server/identity/app/me_routes.py @@ -36,7 +36,7 @@ async def me_set_default_namespace( from server.core.ch import get_client - subject, org = me_callers.caller_identity(request) + subject, org = await me_callers.caller_identity(request) namespace = str(payload.get("namespace") or "").strip() if not namespace: @@ -161,7 +161,7 @@ async def me_sharing_summary(request: Request) -> dict[str, object]: # Auth-off keeps a self-minted bearer's subject so per-subject role lookups # stay accurate on multi-member namespaces. - subject, org = me_callers.caller_identity(request, prefer_bearer_in_dev=True) + subject, org = await me_callers.caller_identity(request) subs = await me_callers.lowered_aliases(org, subject) ch = get_client() diff --git a/server/identity/app/me_service_accounts.py b/server/identity/app/me_service_accounts.py index 23c0cf0232..5234aa2d0f 100644 --- a/server/identity/app/me_service_accounts.py +++ b/server/identity/app/me_service_accounts.py @@ -204,6 +204,7 @@ async def save_token( ) -> None: """List the token in its owner's token list (`loom.service_accounts`). Called under the name's claim.""" + from server.auth.authz import identity_write from server.core.ch import get_client from server.core.versions import next_version @@ -217,76 +218,62 @@ async def save_token( # revoked row from a host running ahead would otherwise replace this one. version = await next_version(ch, "loom.service_accounts", {"org": delegated.org, "name": name}) lineage = {created_by, delegated.creator, *delegated.parent_subjects} - {""} - await ch.insert( - "loom.service_accounts", - [ - { - "org": delegated.org, - "name": name, - "sa_subject": sa_subject, - "namespaces": delegated.namespaces, - "roles": roles, - "ns_roles": json.dumps(ns_roles), - "ttl_days": delegated.ttl_days, - "created_by": created_by, - "created_at": now, - "expires_at": delegated.expires_at(now), - "revoked_at": None, - "revoked_by": "", - "version": version, - "kind": "service_account", - "client": "", - # The gate and the namespace listings read this flag: a token - # reads through its creator's live grants (server/auth/service_accounts.py). - "inherit_creator_grants": True, - "lineage": sorted(lineage), - "saves_to": delegated.saves_to, - } - ], - ) + with identity_write(): + await ch.insert( + "loom.service_accounts", + [ + { + "org": delegated.org, + "name": name, + "sa_subject": sa_subject, + "namespaces": delegated.namespaces, + "roles": roles, + "ns_roles": json.dumps(ns_roles), + "ttl_days": delegated.ttl_days, + "created_by": created_by, + "created_at": now, + "expires_at": delegated.expires_at(now), + "revoked_at": None, + "revoked_by": "", + "version": version, + "kind": "service_account", + "client": "", + # The gate and the namespace listings read this flag: a token + # reads through its creator's live grants (server/auth/service_accounts.py). + "inherit_creator_grants": True, + "lineage": sorted(lineage), + "saves_to": delegated.saves_to, + } + ], + ) -def dev_bearer(request: Request) -> AuthInfo | None: - """With auth off, the principal of a valid bearer the request sends; None - for no bearer or one that does not verify, which auth off ignores. 503 while - a bearer cannot be checked yet, as `verify_bearer` answers.""" - from server.auth.bearer import auth_unavailable, bearer_token, gateway_refused - from server.auth.gateway import is_end_user - from server.auth.jwt import AuthUnavailable, TokenError, verify_token - from server.observability.metrics import record_auth_refusal +async def dev_bearer(request: Request) -> AuthInfo | None: + """A supplied bearer is checked in dev mode too; only absence returns None.""" + from server.auth.bearer import verify_bearer_header - try: - token = bearer_token(request.headers.get("Authorization")) - info = None if token is None else verify_token(token) - except AuthUnavailable as e: - record_auth_refusal(e, transport="http") - raise auth_unavailable(e) from e - except TokenError: - return None - if info is not None and is_end_user(info): - raise gateway_refused() - return info + return await verify_bearer_header(request.headers.get("Authorization")) -def dev_caller(request: Request, params: Mapping[str, object]) -> tuple[str, str]: +async def dev_caller(request: Request, params: Mapping[str, object]) -> tuple[str, str]: """(subject, org) of the caller in dev mode (auth off): a valid bearer's, else the request's `subject` and `org`, else the single user in `self_signup_org`, who holds the default namespace's admin grant (`server.auth.default_namespace`). Every route that saves, lists or revokes a token reads it, so a token minted with auth off is in the list it is revoked from.""" - info = dev_bearer(request) + info = await dev_bearer(request) if info is not None: return info.subject, info.org subject = params.get("subject") or auth_off_subject() return str(subject), str(params.get("org") or settings.self_signup_org) -def dev_parent(request: Request, payload: dict[str, object]) -> AuthInfo: +async def dev_parent(request: Request, payload: dict[str, object]) -> AuthInfo: """The parent a token is delegated from in dev mode (auth off): a flat writer for `dev_caller`, issued now. A bearer only names the owner; its scope and lifetime do not cap the token, as with token.mint.""" - subject, org = dev_caller(request, payload) + subject, org = await dev_caller(request, payload) return AuthInfo( subject=subject, org=org, @@ -339,9 +326,9 @@ async def me_service_accounts_create( """ caller: AuthInfo | None = None # auth off: a mint writes no audit row if settings.auth_actually_required(): - parent = caller = me_callers.require_authed_info(request) + parent = caller = await me_callers.require_authed_info(request) else: - parent = dev_parent(request, payload) + parent = await dev_parent(request, payload) subject, org = parent.subject, parent.org name = token_name(payload.get("name"), default=generated_name(naive_utc_now_seconds())) @@ -437,9 +424,9 @@ async def me_service_accounts_list(request: Request) -> dict[str, object]: from server.core.ch import get_client if settings.auth_actually_required(): - subject, org = me_callers.require_authed_caller(request) + subject, org = await me_callers.require_authed_caller(request) else: - subject, org = dev_caller(request, request.query_params) + subject, org = await dev_caller(request, request.query_params) # Every spelling of the caller, in any case, as grants match a subject. subs = await me_callers.lowered_aliases(org, subject) @@ -535,12 +522,13 @@ async def me_service_accounts_revoke( """Mark an SA as revoked and revoke its subject, so its token stops verifying. Caller must be the creator or admin on at least one of the SA's namespaces.""" + from server.auth.authz import identity_write from server.core.ch import get_client if settings.auth_actually_required(): - subject, org = me_callers.require_authed_caller(request) + subject, org = await me_callers.require_authed_caller(request) else: - subject, org = dev_caller(request, payload) + subject, org = await dev_caller(request, payload) name = str(payload.get("name") or "").strip() if not name: @@ -595,14 +583,15 @@ async def me_service_accounts_revoke( # Revoked already (a retry, a second click): its revocation stands as it was. now = row["revoked_at"] or naive_utc_now_seconds() if not row["revoked_at"]: - await ch.insert( - "loom.service_accounts", - [ - revoked_row( - {**row, "org": org, "name": name}, revoked_at=now, revoked_by=subject - ) - ], - ) + with identity_write(): + await ch.insert( + "loom.service_accounts", + [ + revoked_row( + {**row, "org": org, "name": name}, revoked_at=now, revoked_by=subject + ) + ], + ) # The row keeps no jti, so revoke by `sub` (the sa_subject); its children # die with it through `parent_subjects`. Through `revoke`, so this replica diff --git a/server/identity/app/namespace.py b/server/identity/app/namespace.py index 467b7de31c..b7efa22ec3 100644 --- a/server/identity/app/namespace.py +++ b/server/identity/app/namespace.py @@ -364,6 +364,8 @@ def refuse_unwritable_name(namespace: str) -> None: async def handle_namespace_register( input: NamespaceRegisterInput, ) -> NamespaceRegisterOutput: + from server.auth.authz import identity_write + ch = get_client() caller = resolve_subject() refuse_unwritable_name(input.namespace) @@ -395,7 +397,8 @@ async def handle_namespace_register( now = naive_utc_now_seconds() row = registry_row(input, existing, now) - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) # The claim cap reads which accounts tie a person's spellings from this table. await grant_cache.changed(input.org) # The class this row implies is what the next request in this process runs as — no @@ -643,6 +646,8 @@ async def handle_namespace_delete( async def delete_namespace(org: str, namespace: str) -> NamespaceDeleteOutput: """`namespace.delete`'s write, and `collection.delete`'s once its purge ran.""" + from server.auth.authz import identity_write + ch = get_client() # The sources lease first, as code.repo.remove takes it, so an add sees the delete. async with sources_lease(org, namespace), account_change(org, namespace): @@ -667,7 +672,8 @@ async def delete_namespace(org: str, namespace: str) -> NamespaceDeleteOutput: row["active"] = False # Never behind the row it replaces: on a tie the row inserted last wins. row["updated_at"] = max(now, existing["updated_at"]) - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) if existing.get("kind") == "collection": # The tree last, and on every delete: only it decides access, so a # delete cut short before it is retried (the gate still admits it) diff --git a/server/identity/app/namespace_autoregister.py b/server/identity/app/namespace_autoregister.py index 01c25839cb..0c8484b36f 100644 --- a/server/identity/app/namespace_autoregister.py +++ b/server/identity/app/namespace_autoregister.py @@ -126,6 +126,8 @@ async def ensure_registered( ) -> bool: """Insert the registry row for (org, namespace) if none exists. Returns True when a row was inserted. Never raises.""" + from server.auth.authz import identity_write + if (org, namespace) in _known or reserved_for_sso(namespace): return False # an SSO account's row is single sign-on's to create try: @@ -173,7 +175,8 @@ async def ensure_registered( # Both versions sit below any explicit row's (a register, a restriction), so an # explicit row wins under FINAL, even one another replica wrote a moment ago. row["updated_at"] = OWNERLESS_ROW_VERSION if in_use else AUTO_ROW_VERSION - await ch.insert("loom.namespaces", [row]) + with identity_write(): + await ch.insert("loom.namespaces", [row]) _remember(org, namespace) from server.identity.app.traffic_class_registry import note_registered diff --git a/server/identity/app/namespace_snapshot.py b/server/identity/app/namespace_snapshot.py index c1bb3f9f3b..2db398a78e 100644 --- a/server/identity/app/namespace_snapshot.py +++ b/server/identity/app/namespace_snapshot.py @@ -168,6 +168,7 @@ async def handle_namespace_restore( namespace.""" import time as _time + from server.auth.authz import identity_write from server.observability import audit as _audit _registry.refuse_unwritable_name(input.target_namespace) @@ -253,30 +254,34 @@ async def handle_namespace_restore( # soft-deleted one, which is reactivated with every other column # carried forward. Reuses the probe above. if target_existing is not None and not bool(target_existing.get("active", False)): - await ch.insert( - "loom.namespaces", [{**target_existing, "active": True, "updated_at": now}] - ) + with identity_write(): + await ch.insert( + "loom.namespaces", [{**target_existing, "active": True, "updated_at": now}] + ) await grant_cache.changed(input.org) # the claim cap reads account ties from this table elif target_existing is None: - await ch.insert( - "loom.namespaces", - [ - { - "org": input.org, - "namespace": input.target_namespace, - "owner_email": caller, - "description": (f"Restored from {payload.org}/{payload.source_namespace}"), - "tags": [], - "active": True, - "conflict_threshold": None, - "retention_days": 0, - "warm_days": 0, - "visibility": "private", - "created_at": now, - "updated_at": now, - } - ], - ) + with identity_write(): + await ch.insert( + "loom.namespaces", + [ + { + "org": input.org, + "namespace": input.target_namespace, + "owner_email": caller, + "description": ( + f"Restored from {payload.org}/{payload.source_namespace}" + ), + "tags": [], + "active": True, + "conflict_threshold": None, + "retention_days": 0, + "warm_days": 0, + "visibility": "private", + "created_at": now, + "updated_at": now, + } + ], + ) await _registry.insert_admin_grant(ch, input.org, input.target_namespace, caller) pending = await memory_snapshot.insert_version_rows(rows, ch=ch) if rows else False diff --git a/server/identity/app/password_link.py b/server/identity/app/password_link.py index a824f9cab5..f805b1855a 100644 --- a/server/identity/app/password_link.py +++ b/server/identity/app/password_link.py @@ -115,6 +115,7 @@ async def set_password_page(code: str = "") -> Response: @router.post("/oauth/set-password") async def set_password(request: Request) -> Response: + from server.auth.authz import identity_write from server.auth.passwords import hash_password_off_loop f = await oauth.form_body(request) @@ -149,13 +150,14 @@ async def set_password(request: Request) -> Response: await sso_links.link(email, accounts, "password-link") now = datetime.now(UTC).replace(tzinfo=None, microsecond=0) # Insert-as-replace carrying every other column forward. - await get_client().insert( - "loom.namespaces", - [ - {**r, "password_hash": password_hash, "updated_at": max(now, r["updated_at"])} - for r in rows - ], - ) + with identity_write(): + await get_client().insert( + "loom.namespaces", + [ + {**r, "password_hash": password_hash, "updated_at": max(now, r["updated_at"])} + for r in rows + ], + ) await audit.emit( "auth.password_set", org=org, diff --git a/server/identity/app/scim_groups.py b/server/identity/app/scim_groups.py index cfe69eaa88..68882f5e5c 100644 --- a/server/identity/app/scim_groups.py +++ b/server/identity/app/scim_groups.py @@ -29,7 +29,7 @@ from server.auth.bearer import bearer_token from server.auth.gateway import is_end_user from server.auth.grant_claims import group_change -from server.auth.jwt import AuthInfo, AuthUnavailable, TokenError, verify_token +from server.auth.jwt import AuthInfo, AuthUnavailable, TokenError, authenticate_token from server.auth.scim_routes import ( LAST_INDEX, LIST_SCHEMA, @@ -96,7 +96,7 @@ async def _caller(request: Request) -> _Caller: record_auth_refusal(e, transport="scim") raise ScimError(401, message) from None try: - auth: AuthInfo = verify_token(token) + auth: AuthInfo = await authenticate_token(token) except AuthUnavailable as e: # the revocation list or keyring is not loaded yet raise ScimError(503, "tokens cannot be checked yet; retry") from e except TokenError as e: diff --git a/server/insights/app/admin_routes.py b/server/insights/app/admin_routes.py index c970f52dca..a76f6900bc 100644 --- a/server/insights/app/admin_routes.py +++ b/server/insights/app/admin_routes.py @@ -77,24 +77,24 @@ async def require_admin(request: Request) -> AuthInfo | None: identity provider over a prod-derived `LOOM_AUTH_REQUIRED=true` does not lock the admin surface while every other route is open. """ - return _verified_admin(request, Role.PLATFORM_ADMIN) + return await _verified_admin(request, Role.PLATFORM_ADMIN) async def require_org_admin(request: Request) -> AuthInfo | None: """Bearer + ORG_ADMIN gate for the endpoints that act inside the one org their body names; each handler pins that org to the caller's own (a platform admin passes for any org).""" - return _verified_admin(request, Role.ORG_ADMIN) + return await _verified_admin(request, Role.ORG_ADMIN) -def _verified_admin(request: Request, needed: Role) -> AuthInfo | None: +async def _verified_admin(request: Request, needed: Role) -> AuthInfo | None: """A per-namespace admin (a flat `admin` role, or a scoped token's `ns_roles`) is not deployment or org authority and never passes.""" from server.auth.bearer import missing_bearer, verify_bearer_header + info = await verify_bearer_header(request.headers.get("Authorization")) if not settings.auth_actually_required(): return None - info = verify_bearer_header(request.headers.get("Authorization")) if info is None: raise missing_bearer() if info.has_ns_roles() or not info.has_role(needed): @@ -863,7 +863,7 @@ async def admin_token_mint( creator = info.subject else: # Auth off: a valid bearer names the owner, in its own org; else the single user. - bearer = tokens.dev_bearer(request) + bearer = await tokens.dev_bearer(request) await admit_org(bearer, input.org, "mint a token for") creator = bearer.subject if bearer is not None else auth_off_subject() if input.name is None and "subject" in body: diff --git a/server/knowledge/app/download_routes.py b/server/knowledge/app/download_routes.py index 1996b29630..29c8068a5c 100644 --- a/server/knowledge/app/download_routes.py +++ b/server/knowledge/app/download_routes.py @@ -47,7 +47,7 @@ async def knowledge_download( """ # The HTTP transport's rule: a present bearer is verified whether or not # the deployment requires one; a missing one is a 401 only when auth is on. - auth = verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) + auth = await verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) if auth is None and settings.auth_actually_required(): raise missing_bearer() try: diff --git a/server/knowledge/app/upload_routes.py b/server/knowledge/app/upload_routes.py index bae37b805e..46e37115db 100644 --- a/server/knowledge/app/upload_routes.py +++ b/server/knowledge/app/upload_routes.py @@ -96,7 +96,7 @@ async def knowledge_upload( namespace), as the ops check it; an admitted upload to a restricted namespace leaves the audit row an op leaves. Answers the upload's id, sha256, size, detected format and expiry.""" - auth = verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) + auth = await verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) if auth is None and settings.auth_actually_required(): raise missing_bearer() name = filename.strip().replace("\\", "/").rsplit("/", 1)[-1] diff --git a/server/tasks/app/watch_routes.py b/server/tasks/app/watch_routes.py index 146b424487..5d69cf0881 100644 --- a/server/tasks/app/watch_routes.py +++ b/server/tasks/app/watch_routes.py @@ -111,7 +111,7 @@ async def event_stream( await wait(key, min(beat_in, deadline - now)) -def _bearer_auth(request: Request) -> Any: +async def _bearer_auth(request: Request) -> Any: """The verified principal — the HTTP transport's rule: a bearer that is present is verified whether or not the deployment requires one (a CI server runs with LOOM_AUTH_REQUIRED=false and its tests still mint @@ -119,7 +119,7 @@ def _bearer_auth(request: Request) -> Any: and None (no principal, every scope check answers yes) otherwise.""" from server.auth.bearer import missing_bearer, verify_bearer_header - auth = verify_bearer_header(request.headers.get("Authorization")) + auth = await verify_bearer_header(request.headers.get("Authorization")) if auth is None and settings.auth_actually_required(): raise missing_bearer() return auth @@ -198,7 +198,7 @@ async def task_watch( # Not a registry op, so no transport tags this route's reads; the stream's # iteration inherits the tag. set_query_tag("task.watch") - auth = _bearer_auth(request) + auth = await _bearer_auth(request) set_current_auth(auth) key, inbox_filter, subject = await _resolve_key( org, inbox, bool(posted_by_me), from_subject, auth diff --git a/server/tracker/app/registry.py b/server/tracker/app/registry.py index a527e5db9d..13ffff1d94 100644 --- a/server/tracker/app/registry.py +++ b/server/tracker/app/registry.py @@ -31,6 +31,8 @@ async def ensure_tracker_registered( join_policy: str = "", ) -> bool: """The caller holds the namespace's account claim.""" + from server.auth.authz import identity_write + key = (org, namespace) if key in _DONE: return False @@ -46,38 +48,40 @@ async def ensure_tracker_registered( if restricted and str(row.get("classification") or "standard") != "restricted": narrowed = dict(row) narrowed.update(classification="restricted", visibility="private", updated_at=now) - await client.insert("loom.namespaces", [narrowed]) + with identity_write(): + await client.insert("loom.namespaces", [narrowed]) from server.identity.app.namespace_classification import note_restricted note_restricted(org, namespace) logger.warning("qa namespace %s/%s narrowed to restricted", org, namespace) _DONE.add(key) return False - await client.insert( - "loom.namespaces", - [ - { - "org": org, - "namespace": namespace, - "owner_email": "", - "description": ( - "tracker record, read live (issues, PRs, triage, risk; " - "read-only reference data)" - ), - "tags": ["tracker"], - "active": True, - "kind": "tracker", - "visibility": "private", - "classification": "restricted" if restricted else "standard", - # A public tracker namespace is joined, not granted — any - # principal in the org joins it directly; a restricted one asks - # (a registrant may choose `approval` for a public one) - "join_policy": "approval" if restricted else (join_policy or "open"), - "created_at": now, - "updated_at": now, - } - ], - ) + with identity_write(): + await client.insert( + "loom.namespaces", + [ + { + "org": org, + "namespace": namespace, + "owner_email": "", + "description": ( + "tracker record, read live (issues, PRs, triage, risk; " + "read-only reference data)" + ), + "tags": ["tracker"], + "active": True, + "kind": "tracker", + "visibility": "private", + "classification": "restricted" if restricted else "standard", + # A public tracker namespace is joined, not granted — any + # principal in the org joins it directly; a restricted one asks + # (a registrant may choose `approval` for a public one) + "join_policy": "approval" if restricted else (join_policy or "open"), + "created_at": now, + "updated_at": now, + } + ], + ) if restricted: from server.identity.app.namespace_classification import note_restricted diff --git a/server/transport/gates.py b/server/transport/gates.py index e6251c6ad3..276f188716 100644 --- a/server/transport/gates.py +++ b/server/transport/gates.py @@ -22,8 +22,8 @@ authorized. A refused default (a member's namespace, or the registry unread) sets none and is kept for the request, so the handler refuses too. - `authorize`: the refusal of `@me` and of a mirror's name first, then the cell's - tenancy check (`tenancy.ensure_held`), then the refusal of a revoked subject a - trusted caller acts for (`authz.check_acted_for`), first-contact provisioning, the + complete person and acted-for check (`authz.check_acted_for`), then the tenancy + check (`tenancy.ensure_held`), first-contact provisioning, the role + namespace gate (`authorize_op`, raising PermissionError / IdentityIncompleteError; a scoped token in its own org gets a refused default's own refusal instead), then the restricted-access audit row and the first-write @@ -287,11 +287,14 @@ async def authorize( raise PermissionError(MIRROR_REFUSED) # A cell serves only the orgs it holds: checked before anything below provisions # or writes for the call (docs/architecture/cells.md). + from server.auth.authz import check_acted_for + + await check_acted_for(auth, target.org) await tenancy.ensure_held(target.org, auth) if auth is None: _note_admitted(target) return - from server.auth.authz import check_acted_for, scope_flags + from server.auth.authz import scope_flags from server.auth.gateway import is_end_user from server.auth.personal_namespace import ensure_personal_namespace_cached from server.code.app.member_writes import authorize_op @@ -310,7 +313,6 @@ async def authorize( # A subject a trusted caller may not act for (revoked, deprovisioned, or an `sa:` # subject naming no active bot) is refused before anything is provisioned for it # (`check_acted_for` admits every other call). - await check_acted_for(auth, target.org) # The personal namespace is the token's own org's: on a cell, written only # where that org is held (a platform admin of another org acting here is not). if auth.org == target.org or await tenancy.is_held(auth.org): diff --git a/server/transport/http.py b/server/transport/http.py index 28cca4ae65..f0cf4d55d1 100644 --- a/server/transport/http.py +++ b/server/transport/http.py @@ -126,7 +126,7 @@ def build_router(components: dict[str, Any] | None = None) -> APIRouter: return router -def _extract_auth(request: Request) -> AuthInfo | None: +async def _extract_auth(request: Request) -> AuthInfo | None: """Pull a bearer token from the Authorization header and verify it. Returns None if no token is present (allowed only when auth_required=False). @@ -134,7 +134,7 @@ def _extract_auth(request: Request) -> AuthInfo | None: tokens when auth_required=True. """ # Counts its own refusals; an end-user token is served here. - auth = verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) + auth = await verify_bearer_header(request.headers.get("Authorization"), acts_for_end_users=True) if auth is None: if settings.auth_actually_required(): raise missing_bearer() @@ -759,7 +759,7 @@ async def endpoint(request: Request, response: Response) -> Any: # code-trace row (the body and validation refusals write both), # learns no field errors, and never touches a tenant's rate-limit # bucket by naming its namespace. - call.auth = auth = _extract_auth(request) + call.auth = auth = await _extract_auth(request) # Bound before the body is read, as MCP binds it before validation: # a refusal's ledger row reads the token's org and pins from it. from server.auth.context import set_current_auth diff --git a/server/transport/mcp.py b/server/transport/mcp.py index 3bffa6810e..3899c774d1 100644 --- a/server/transport/mcp.py +++ b/server/transport/mcp.py @@ -372,11 +372,11 @@ async def mcp_endpoint(request: Request, response: Response) -> Any: # schema parse and surfaces as an opaque zod error. mode = _effective_auth_mode() try: - auth = _verify_bearer(request) + auth = await _verify_bearer(request) except PermissionError as e: return _not_on_mcp(response, payload, e) except TokenError as e: - if mode != "open": + if mode != "open" or e.reason == "provisioning": return _token_refused(request, response, payload, e) # Open mode falls back to no-auth rather than 401 (so bridges # with mis-templated headers still work). Warn-log it — silent @@ -461,11 +461,11 @@ async def mcp_session_end(request: Request, response: Response) -> Any: mode = _effective_auth_mode() try: - auth = _verify_bearer(request) + auth = await _verify_bearer(request) except PermissionError as e: return _not_on_mcp(response, None, e) except TokenError as e: - if mode != "open": + if mode != "open" or e.reason == "provisioning": return _token_refused(request, response, None, e) auth = None if mode != "open" and auth is None: diff --git a/server/transport/mcp_auth.py b/server/transport/mcp_auth.py index 2b7f318d01..d501e6da53 100644 --- a/server/transport/mcp_auth.py +++ b/server/transport/mcp_auth.py @@ -208,7 +208,7 @@ async def _plane_namespace( return nr.unknown_result(ns, nr.suggest(ns, nr.suggestable(org, auth.can_access))), False -def _verify_bearer(request: Request) -> AuthInfo | None: +async def _verify_bearer(request: Request) -> AuthInfo | None: """Verify the JWT on an MCP request. The bearer comes from the ``Authorization: Bearer `` header only. A @@ -226,12 +226,12 @@ def _verify_bearer(request: Request) -> AuthInfo | None: set_code_scope(request.headers.get("x-loom-code-scope")) from server.auth.bearer import bearer_token from server.auth.gateway import is_end_user - from server.auth.jwt import verify_token + from server.auth.jwt import authenticate_token token = bearer_token(request.headers.get("authorization")) if token is None: return None - auth = verify_token(token) + auth = await authenticate_token(token) if is_end_user(auth): # An end user's service calls the HTTP API, which maps `@me` and its downloads. raise PermissionError("an end-user token calls the HTTP API (/v1/), not MCP") diff --git a/server/transport/tenancy_gate.py b/server/transport/tenancy_gate.py index 8ed652de05..d8c51a1c61 100644 --- a/server/transport/tenancy_gate.py +++ b/server/transport/tenancy_gate.py @@ -69,11 +69,11 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if not acts_in_token_org(path): await self.app(scope, receive, send) return - from server.auth.jwt import AuthUnavailable, TokenError, verify_token + from server.auth.jwt import AuthUnavailable, TokenError, authenticate_token try: token = _bearer(scope) - info = verify_token(token) if token is not None else None + info = await authenticate_token(token) if token is not None else None except AuthUnavailable as e: # Cannot judge the token yet: refused, never passed on unchecked. The caller # gets the error's kind; the text is logged. @@ -99,9 +99,17 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: return try: org = acting_org(path, info) + from server.auth.authz import check_acted_for + + await check_acted_for(info, org) # The sign-up org a route falls back to is not the token's: no principal, so # it is recorded or refused, never placed, as for a sign-up. await tenancy.ensure_held(org, info if org == info.org else None) + except PermissionError as e: + await _refuse( + scope, receive, send, path, 401, {"code": "AUTH_INVALID", "message": str(e)}, None + ) + return except LoomServerError as e: await _refuse(scope, receive, send, path, e.status_code, e.to_detail(), e.headers()) return diff --git a/tests/integration/scim_session_previous.py b/tests/integration/scim_session_previous.py new file mode 100644 index 0000000000..81ee6e488d --- /dev/null +++ b/tests/integration/scim_session_previous.py @@ -0,0 +1,76 @@ +"""The previous build's session serializers for rolling-upgrade tests.""" + +from __future__ import annotations + +from typing import Any, cast + +from server.auth import claims +from server.auth.oauth_store import UPSTREAM_PURPOSE, Device, Grant +from server.core.secrets_at_rest import seal, unseal + +_CODE_FIELDS = ( + "client_id", + "redirect_uri", + "code_challenge", + "subject", + "org", + "namespaces", + "roles", + "exp", + "since", + "family", +) + + +_DEVICE_FIELDS = ( + "id", + "client_id", + "subject", + "org", + "namespaces", + "roles", + "exp", + "family", + "user_code", + "status", + "address", + "ticket", +) + + +def _record(g: Grant | Device, fields: tuple[str, ...], spent: bool) -> dict[str, Any]: + """A grant as kept in Keeper: its IdP token sealed, and a nonce so no two + versions are equal (claims.swap relies on it).""" + record: dict[str, Any] = {f: g[f] for f in fields} # type: ignore[literal-required] + record["upstream_refresh"] = seal(UPSTREAM_PURPOSE, g["upstream_refresh"]) + return record | {"spent": spent, "nonce": claims.nonce()} + + +def _code(r: dict[str, Any]) -> Grant: + g = {f: r[f] for f in _CODE_FIELDS} + return cast(Grant, g | {"upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"])}) + + +def _device(r: dict[str, Any]) -> Device: + d = {f: r[f] for f in _DEVICE_FIELDS} + return cast(Device, d | {"upstream_refresh": unseal(UPSTREAM_PURPOSE, r["upstream_refresh"])}) + + +_REFRESH_TYPE = "session_refresh" + + +def _refresh_row(rid: str, g: Grant, version: int) -> dict[str, Any]: + return { + "record_type": _REFRESH_TYPE, + "id": rid, + "client_id": g["client_id"], + "subject": g["subject"], + "org": g["org"], + "namespaces": list(g["namespaces"]), + "roles": list(g["roles"]), + "expires_at": int(g["exp"]), + "family": g["family"], + "generation": g.get("generation", 0), + "upstream_refresh": seal(UPSTREAM_PURPOSE, g["upstream_refresh"]), + "version": version, + } diff --git a/tests/integration/test_scim_erasure_live_ch.py b/tests/integration/test_scim_erasure_live_ch.py new file mode 100644 index 0000000000..e21fb1d73f --- /dev/null +++ b/tests/integration/test_scim_erasure_live_ch.py @@ -0,0 +1,292 @@ +"""Actual erasure rewrites release a native child's cached account-size refusal.""" + +from __future__ import annotations + +import json +import time +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any + +import pytest + +from server.auth import delegation, revocation, scim, scim_people, scim_routes +from server.auth.grant_claims import service_account_change +from server.auth.jwt import AuthInfo +from server.core import ch +from server.core.principals import USER, personal_name, principal_id +from server.erasure.app import saga_steps +from server.identity.api.models import NamespaceRegisterInput +from server.identity.app import me_service_accounts +from server.identity.app.namespace import registry_row +from server.identity.store import collections as collections_repo +from server.operations.base import Role +from server.settings import settings +from tests.integration._resources import unique_org +from tests.integration.test_scim_person_live_ch import People +from tests.integration.test_scim_person_live_ch import people as people + +pytestmark = pytest.mark.usefixtures("ch_target_required") + +# Integration conftests do not replace this policy. Check that before delegation. +_PARENT_POLICY = delegation._refuse_a_revoked_parent + + +def directory_row(org: str, namespace: str, owner: str, *, ordinary: bool) -> dict[str, Any]: + row = registry_row( + NamespaceRegisterInput( + org=org, namespace=namespace, owner_email=owner, + kind="memory" if ordinary else None, description="directory description", + ), + None, + datetime.now(UTC).replace(microsecond=0), + ) + assert row["contact_email"] == row["password_hash"] == "" + return row + + +async def stored_row(client: ch.CHClient, table: str, org: str, namespace: str) -> dict[str, Any]: + assert table in ("loom.namespaces", "loom.collections") + with ch.own_write_reads(): + row = await client.query_one( + f"SELECT * FROM {table} FINAL WHERE org = {{org:String}} " + "AND namespace = {ns:String} LIMIT 1", + params={"org": org, "ns": namespace}, + ) + assert row is not None + return row + + +async def cold_decision(auth: AuthInfo, expected_accounts: int, refused: bool) -> None: + extra = scim_people._roots( + auth.subject, "", (*auth.parent_subjects, auth.creator), auth.org + ) + seconds = scim_people._decision_seconds() + decision = scim_people._Decision(auth.org, auth.subject, "", scim_people._now() + seconds) + with ch.own_write_reads(), ch.query_tag("test.scim_erasure.cold"): + try: + result = await decision.refuses((auth.subject, *extra)) + except scim_people._Refused: + result = True + assert result is refused + assert decision.accounts == expected_accounts + assert decision.transferred < scim_people.MAX_BYTES + + +@pytest.mark.parametrize("kind", ["personal", "collection"]) +async def test_real_erasure_rewrite_rechecks_a_native_child_after_account_count_shrinks( + people: People, monkeypatch: pytest.MonkeyPatch, kind: str +) -> None: + client, org = people.client, people.org + user_id = f"erasure-{people.run}@example.test" + owner = USER + user_id + assert principal_id(owner, USER) == user_id + target = personal_name(org, owner) if kind == "personal" else "col-erasure-target" + other_org = unique_org("scim-erasure-other") + monkeypatch.setattr(settings, "grant_cache_ttl_seconds", 240) + + # Ordinary operator-registerable memory rows do not add collection claims. + ordinary = [ + directory_row(org, f"memory-erasure-{i}", owner, ordinary=True) for i in range(1024) + ] + assert len({r["namespace"] for r in ordinary}) == 1024 + assert all(r["kind"] == "memory" and r["active"] for r in ordinary) + target_row = directory_row(org, target, owner, ordinary=False) + assert target_row["kind"] == kind + if kind == "collection": + # collection.create fills the title after the same registry builder. + target_row["title"] = "Erasure collection" + # Same namespace and owner ensure the collection rewrite needs its org filter. + # The foreign personal-name row is an operator-registerable memory row. + foreign_row = directory_row(other_org, target, owner, ordinary=kind == "personal") + await client.insert( + "loom.namespaces", [*ordinary, target_row, foreign_row], + settings={"async_insert": 0, "wait_for_async_insert": 1}, + ) + if kind == "collection": + await collections_repo.write(collections_repo.collection_row(org, target, owner=owner)) + await collections_repo.write( + collections_repo.collection_row(other_org, target, owner=owner) + ) + before = await stored_row(client, "loom.namespaces", org, target) + foreign_before = await stored_row(client, "loom.namespaces", other_org, target) + tree_before = ( + await stored_row(client, "loom.collections", org, target) + if kind == "collection" else None + ) + foreign_tree_before = ( + await stored_row(client, "loom.collections", other_org, target) + if kind == "collection" else None + ) + _, parsed, _ = scim_routes.user_name_and_email( + {"userName": owner, "emails": [{"value": owner, "type": "work", "primary": True}]} + ) + assert parsed == owner + # The SCIM table has no org; use the existing fixture's exact-id cleanup. + people.emails.append(owner) + await people.provision(owner) + + revocation_reads: list[tuple[dict[str, Any], Any]] = [] + account_counts: list[int] = [] + original_all, original_one = client.query_all, client.query_one + + async def read_all(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + if params is not None: + assert scim_people._encoded(params) < scim_people.MAX_BYTES + answer = await original_all(sql, params=params, **kwargs) + if "FROM loom.revoked_tokens FINAL" in sql and "jti IN {keys:Array(String)}" in sql: + revocation_reads.append((dict(params or {}), answer)) + return answer + + async def read_one(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + if params is not None: + assert scim_people._encoded(params) < scim_people.MAX_BYTES + answer = await original_one(sql, params=params, **kwargs) + if "AS account_count" in sql or "AS bounded_creator" in sql: + applied = ch.statement_settings(sql) + assert 0 < applied["max_execution_time"] <= scim_people.SCIM_DECISION_S + if "AS account_count" in sql: + assert answer is not None and int(answer["account_bytes"]) == 0 + count = int(answer["account_count"]) + if count > 1024: + assert answer["accounts"] == [] + account_counts.append(count) + return answer + + monkeypatch.setattr(client, "query_all", read_all) + monkeypatch.setattr(client, "query_one", read_one) + assert delegation._refuse_a_revoked_parent is _PARENT_POLICY + assert _PARENT_POLICY.__module__ == "server.auth.delegation" + assert _PARENT_POLICY.__name__ == "_refuse_a_revoked_parent" + parent = AuthInfo( + subject=owner, org=org, namespaces=(), roles=(Role.VIEWER,), + is_service_account=False, impersonated_by="https://issuer.example.test", + read_scope="own_user", read_user_id=user_id, + token_id="parent-" + people.run, issued_at=int(time.time()), + ) + child = await delegation.delegate( + parent, org=org, requested={"*": None}, scoped=False, + ttl_seconds=300, is_service_account=True, + ) + expected_keys = sorted( + key for subject in (owner, parent.token_id) + for key in (subject, revocation.CUTOFF_PREFIX + subject, revocation.SCIM_PREFIX + subject) + ) + assert revocation_reads == [({"keys": expected_keys}, [])] + name = "erasure-child-" + people.run + subject = me_service_accounts.new_token_subject(org, name, owner) + async with service_account_change(org, name): + await me_service_accounts.refuse_taken_name(org, name, owner) + await me_service_accounts.save_token( + name, subject, child, created_by=owner, now=datetime.now(UTC) + ) + auth = child.as_auth(subject) + assert auth.creator == owner and owner in auth.parent_subjects + assert auth.impersonated_by == "" and auth.is_service_account + persisted = await client.query_one( + "SELECT created_by, lineage, revoked_at FROM loom.service_accounts FINAL " + "WHERE org = {org:String} AND name = {name:String} LIMIT 1", + params={"org": org, "name": name}, + ) + assert persisted is not None and persisted["created_by"] == owner + assert owner in persisted["lineage"] and persisted["revoked_at"] is None + + await cold_decision(auth, 1025, True) + assert account_counts[-1] == 1025 + warmed_at = time.monotonic() + assert await scim.refuses_auth(auth) is True + reads = len(people.results) + assert await scim.refuses_auth(auth) is True + assert len(people.results) == reads, "the pre-write denial must actually be warm" + + # Observe real claims and real SQL; each wrapper forwards to its original. + account_claims: list[tuple[str, str]] = [] + tree_claims: list[str] = [] + held_accounts: set[tuple[str, str]] = set() + held_trees: set[str] = set() + writes: list[str] = [] + original_account, original_tree = saga_steps.account_change, saga_steps.collections_change + original_execute = client.execute + + @asynccontextmanager + async def account_claim(claim_org: str, namespace: str): + async with original_account(claim_org, namespace): + account_claims.append((claim_org, namespace)) + held_accounts.add((claim_org, namespace)) + try: + yield + finally: + held_accounts.remove((claim_org, namespace)) + + @asynccontextmanager + async def tree_claim(claim_org: str): + async with original_tree(claim_org): + tree_claims.append(claim_org) + held_trees.add(claim_org) + try: + yield + finally: + held_trees.remove(claim_org) + + async def execute(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + if sql.startswith("INSERT INTO loom.") and "SELECT * REPLACE" in sql: + assert held_accounts == {(org, target)} + assert held_trees == ({org} if kind == "collection" else set()) + assert "FROM (SELECT * FROM " in sql and " FINAL WHERE " in sql + assert params is not None and params["org"] == org + writes.append(sql) + return await original_execute(sql, params=params, **kwargs) + + monkeypatch.setattr(saga_steps, "account_change", account_claim) + monkeypatch.setattr(saga_steps, "collections_change", tree_claim) + monkeypatch.setattr(client, "execute", execute) + if kind == "personal": + assert await saga_steps._step_retire_personal_collection_by_user(org, user_id) == 1 + else: + assert await saga_steps._step_forget_collection_owner_by_user(org, user_id) == 2 + assert account_claims == [(org, target)] + assert tree_claims == ([org] if kind == "collection" else []) + assert len(writes) == (1 if kind == "personal" else 2) + assert not held_accounts and not held_trees + + after = await stored_row(client, "loom.namespaces", org, target) + expected = dict(before, owner_email="", updated_at=after["updated_at"]) + if kind == "personal": + expected.update(active=False, description="", title="") + assert after["updated_at"] > before["updated_at"] + assert after == expected, "the namespace rewrite must carry every other persisted field" + assert await stored_row(client, "loom.namespaces", other_org, target) == foreign_before + if kind == "collection": + assert tree_before is not None + tree_after = await stored_row(client, "loom.collections", org, target) + assert tree_after["version"] > tree_before["version"] + assert tree_after == dict(tree_before, owner="", version=tree_after["version"]) + assert ( + await stored_row(client, "loom.collections", other_org, target) + ) == foreign_tree_before + owner_rows = await client.query_one( + "SELECT count() AS n, countIf(kind = 'memory') AS ordinary " + "FROM loom.namespaces FINAL WHERE org = {org:String} AND owner_email = {owner:String} " + "AND ifNull(active, true) = true", + params={"org": org, "owner": owner}, + ) + assert owner_rows == {"n": 1024, "ordinary": 1024} + await cold_decision(auth, 1024, False) + assert account_counts[-1] == 1024 + assert time.monotonic() - warmed_at < settings.grant_cache_ttl_seconds + assert await scim.refuses_auth(auth) is False, ( + "the same warm memo must observe the actual rewrite" + ) + reads = len(people.results) + assert await scim.refuses_auth(auth) is False + assert len(people.results) == reads, "the new qualifying decision must also be memoized" + claim_keys = [ + "service_account:" + json.dumps([org, name]), + "account:" + json.dumps([org, target]), + ] + if kind == "collection": + claim_keys.append("collections:" + json.dumps(org)) + assert await client.query_all( + "SELECT name FROM loom._migration_lock WHERE name IN {names:Array(String)}", + params={"names": claim_keys}, + ) == [] diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py new file mode 100644 index 0000000000..8af632508f --- /dev/null +++ b/tests/integration/test_scim_person_live_ch.py @@ -0,0 +1,328 @@ +"""Person decisions use bounded projections on the actual ClickHouse client.""" + +from __future__ import annotations + +import json +import time +import uuid +from collections.abc import AsyncIterator, Callable +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest + +from server.auth import authz, scim, service_accounts +from server.core import cell, ch +from server.settings import settings +from tests.integration._resources import unique_org + +pytestmark = pytest.mark.usefixtures("ch_target_required") + + +class People: + def __init__(self, client: ch.CHClient): + self.client = client + self.org = unique_org("scim-person") + self.run = uuid.uuid4().hex[:10] + self.emails: list[str] = [] + self.bots: list[str] = [] + self.results: list[tuple[str, dict[str, Any], Any]] = [] + + def email(self, label: str) -> str: + email = f"{label}-{self.run}@example.test" + self.emails.append(email) + return email + + async def account(self, owner: str, email: str, *, password: bool = False) -> str: + namespace = "account-" + uuid.uuid4().hex[:12] + now = datetime.now(UTC) + await self.client.insert( + "loom.namespaces", + [ + { + "org": self.org, + "namespace": namespace, + "owner_email": owner, + "contact_email": email, + "password_hash": "unit-hash" if password else "", + "active": True, + "created_at": now, + "updated_at": now, + } + ], + ) + return namespace + + async def provision( + self, email: str, *, active: bool = True, deleted: bool = False, work: tuple[str, ...] = () + ) -> None: + now = datetime.now(UTC) + await self.client.insert( + "loom.scim_users", + [ + { + "id": scim.user_id(email), + "user_name": email, + "email": email, + "emails": "[]", + "addresses": [email, *work], + "work_addresses": list(work), + "active": active, + "deleted": deleted, + "subjects": [], + "created_at": now, + "updated_at": now, + "version": time.time_ns(), + } + ], + ) + + async def personal( + self, + subject: str, + creator: str, + *, + lineage: list[str] | None = None, + expires_at: datetime | None = None, + revoked_at: datetime | None = None, + ) -> None: + await self.client.insert( + "loom.service_accounts", + [ + { + "org": self.org, + "name": subject.removeprefix("sa:"), + "sa_subject": subject, + "namespaces": [], + "roles": [], + "created_by": creator, + "created_at": datetime.now(UTC), + "lineage": lineage or [], + "inherit_creator_grants": True, + "kind": "personal", + "expires_at": expires_at, + "revoked_at": revoked_at, + "version": time.time_ns(), + } + ], + ) + + async def refused(self, subject: str, **kwargs: Any) -> bool: + authz.forget_ended("identity", "") + return await scim.refuses_subject(subject, org=self.org, **kwargs) + + +@pytest.fixture +async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: + client = ch.get_client() + await client.connect() + made = People(client) + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", "unit-native-person-policy") + monkeypatch.setattr(cell, "mode", lambda: "enforced") + monkeypatch.setattr(cell, "deployment_id", lambda: "unit-native-person-deployment") + monkeypatch.setattr(authz, "_ENDED", {}) + monkeypatch.setattr(service_accounts, "_creators", {}) + query_all, query_one = client.query_all, client.query_one + + async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + answer = await query_all(sql, params=params, **kwargs) + made.results.append((sql, ch.statement_settings(sql), answer)) + return answer + + async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + answer = await query_one(sql, params=params, **kwargs) + made.results.append((sql, ch.statement_settings(sql), answer)) + return answer + + monkeypatch.setattr(client, "query_all", all_rows) + monkeypatch.setattr(client, "query_one", one_row) + try: + yield made + finally: + if made.emails: + await client.execute( + "ALTER TABLE loom.scim_users DELETE WHERE id IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + params={"ids": [scim.user_id(e) for e in made.emails]}, + ) + if made.bots: + await client.execute( + "ALTER TABLE loom.machine_identities DELETE WHERE name IN {names:Array(String)} " + "SETTINGS mutations_sync = 2", + params={"names": made.bots}, + ) + await client.disconnect() + + +async def test_verified_reverse_owner_still_requires_its_own_email(people: People) -> None: + owner, contact = people.email("owner"), people.email("contact") + await people.account(owner, contact) + await people.provision(contact) + assert await people.refused(contact) + await people.provision(owner, work=(contact,)) + assert not await people.refused(contact) + await people.provision(owner, deleted=True) + assert await people.refused(contact) + + +async def test_password_contact_only_refuses_until_a_verified_pair_exists(people: People) -> None: + contact = people.email("password") + namespace = await people.account("owner", contact, password=True) + await people.provision(contact) + assert await people.refused("owner") + await people.client.insert( + "loom.sso_links", + [ + { + "email": contact, + "org": people.org, + "namespace": namespace, + "linked": 1, + "linked_by": "test", + "linked_at": datetime.now(UTC), + "version": time.time_ns(), + } + ], + ) + assert not await people.refused("owner") + + +async def test_owner_ambiguity_is_not_hidden_by_limit(people: People) -> None: + contact = people.email("shared") + await people.account("first-owner", contact) + await people.account("second-owner", contact) + await people.provision(contact) + assert await people.refused(contact) + + +@pytest.mark.parametrize("count,refused", [(1024, False), (1025, True)]) +async def test_native_contact_aggregate_does_not_accept_a_truncated_prefix( + people: People, count: int, refused: bool +) -> None: + email = people.email("bounded") + now = datetime.now(UTC) + await people.client.insert( + "loom.namespaces", + [ + { + "org": people.org, + "namespace": f"account-{i}", + "owner_email": "owner", + "contact_email": email, + "active": True, + "created_at": now, + "updated_at": now, + } + for i in range(count) + ], + ) + await people.provision(email) + assert await people.refused("owner") is refused + aggregate = next(row for sql, _, row in people.results if "AS account_count" in sql) + assert aggregate["account_count"] == count + assert len(aggregate["accounts"]) == (0 if refused else count) + + +async def test_native_contact_bytes_return_only_overflow_metadata(people: People) -> None: + await people.account("owner", "x" * 65537) + assert await people.refused("owner") + aggregate = next(row for sql, _, row in people.results if "AS account_count" in sql) + assert aggregate["account_bytes"] == 65537 + assert aggregate["accounts"] == [] + assert len(json.dumps(aggregate).encode()) < 1024 + + +async def test_work_alias_lookup_transfers_only_the_requested_intersection(people: People) -> None: + owner, contact = people.email("owner"), people.email("work") + await people.account("owner", owner) + await people.account("owner", contact, password=True) + unrelated = tuple(f"unused-{n}@example.test" for n in range(3000)) + await people.provision(owner, work=(*unrelated, contact)) + assert not await people.refused("owner") + alias = next(row for sql, _, row in people.results if "groupUniqArray" in sql) + assert alias == {"qualified": [contact]} + assert len(json.dumps(alias).encode()) < 256 + + +@pytest.mark.parametrize("lineage_count,refused", [(1023, False), (1024, True)]) +async def test_native_repeated_edges_share_the_aggregate_budget( + people: People, lineage_count: int, refused: bool +) -> None: + creator = people.email("creator") + await people.provision(creator) + await people.personal("sa:personal:one", creator, lineage=[creator] * lineage_count) + assert await people.refused("sa:personal:one") is refused + + +async def test_native_fresh_reader_does_not_reuse_the_inheritance_cache(people: People) -> None: + creator = people.email("creator") + await people.provision(creator) + await people.personal("sa:personal:one", creator) + cached = await service_accounts.sa_creator(people.org, "sa:personal:one") + assert cached is not None and cached.subject == creator + await people.personal("sa:personal:one", creator, revoked_at=datetime.now(UTC)) + assert await people.refused("sa:personal:one") + current = await service_accounts._creator_row(people.org, "sa:personal:one") + assert current is not None and current["revoked_at"] is not None + + +async def test_native_fresh_reader_projects_an_oversized_creator_to_metadata( + people: People, +) -> None: + await people.personal("sa:personal:one", "x" * 65537) + assert await people.refused("sa:personal:one") + current = await service_accounts._creator_row(people.org, "sa:personal:one") + assert current is not None and current["created_by"] == "" + assert current["subject_bytes"] == 65537 + assert current["lineage"] == [] + assert await service_accounts.sa_creator(people.org, "sa:personal:one") is None + + +@pytest.mark.parametrize("expired", [False, True]) +async def test_native_expiry_uses_the_current_service_account_row( + people: People, expired: bool +) -> None: + creator = people.email("creator") + await people.provision(creator) + expires = datetime.now(UTC) + timedelta(minutes=-1 if expired else 1) + await people.personal("sa:personal:one", creator, expires_at=expires) + assert await people.refused("sa:personal:one") is expired + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +async def test_actual_client_accepts_finite_remaining_statement_budgets( + people: People, monkeypatch: pytest.MonkeyPatch, read_timeout: float +) -> None: + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + email = people.email("budget") + await people.account("owner", email) + await people.provision(email) + people.results.clear() + with ch.statement_timeout(1.75): + assert not await people.refused("owner") + assert len(people.results) >= 3 + for _, applied, _ in people.results: + assert 0 < applied["max_execution_time"] <= 1.75 + + +async def test_real_person_queries_report_their_physical_work( + people: People, record_property: Callable[[str, Any], None] +) -> None: + email = people.email("query-log") + await people.account("owner", email) + await people.provision(email) + before = await people.client.query_one("SELECT now64(6) AS started") + assert before is not None + assert not await people.refused("owner") + await people.client.execute("SYSTEM FLUSH LOGS") + rows = await people.client.query_all( + "SELECT query_duration_ms, read_rows, read_bytes, memory_usage FROM system.query_log " + "WHERE type = 'QueryFinish' AND log_comment = 'auth.scim_people' " + "AND event_time_microseconds >= {started:DateTime64(6)} " + "ORDER BY event_time_microseconds LIMIT 20", + params={"started": before["started"]}, + ) + assert len(rows) >= 3 + assert any(int(row["read_rows"]) > 0 for row in rows) + record_property("scim_person_query_log", json.dumps(rows, default=str)) diff --git a/tests/integration/test_scim_sessions_live_ch.py b/tests/integration/test_scim_sessions_live_ch.py new file mode 100644 index 0000000000..524d7d6ffd --- /dev/null +++ b/tests/integration/test_scim_sessions_live_ch.py @@ -0,0 +1,279 @@ +"""Verified sign-in evidence survives real refresh and Keeper transitions.""" + +from __future__ import annotations + +import json +import time +import uuid +from collections.abc import AsyncIterator +from typing import Any + +import pytest + +from server.auth import claims, oauth, oauth_store +from server.auth.login import SignInUnavailable +from server.auth.oauth_store import ClickHouseStore, Device, Grant, token_id +from server.core.secrets_at_rest import unseal +from server.identity.app.claim_release import release_claim +from server.settings import settings +from tests.integration import scim_session_previous as previous +from tests.integration.test_scim_person_live_ch import People +from tests.integration.test_scim_person_live_ch import people as people + +pytestmark = pytest.mark.usefixtures("ch_target_required") + + +class Sessions(ClickHouseStore): + def __init__(self, people: People): + self.people = people + self.keys: set[str] = set() + self.families: set[str] = set() + + async def _swap(self, key: str, old: str | None, new: str, expires_at: int) -> str | None: + self.keys.add(key) + if key.startswith("family:"): + self.families.add(key.removeprefix("family:")) + return await super()._swap(key, old, new, expires_at) + + def grant(self, *, email: str = "") -> Grant: + return { + "client_id": "client-" + uuid.uuid4().hex, + "redirect_uri": "", + "code_challenge": "", + "subject": "owner", + "org": self.people.org, + "namespaces": ["account"], + "roles": ["writer"], + "exp": int(time.time()) + 600, + "since": int(time.time()), + "family": uuid.uuid4().hex, + "upstream_refresh": "", + "verified_email": email, + } + + def make_device(self, grant: Grant) -> Device: + return { + "id": token_id(uuid.uuid4().hex), + "client_id": grant["client_id"], + "subject": grant["subject"], + "org": grant["org"], + "namespaces": list(grant["namespaces"]), + "roles": list(grant["roles"]), + "exp": grant["exp"], + "family": grant["family"], + "upstream_refresh": grant["upstream_refresh"], + "verified_email": grant.get("verified_email", ""), + "user_code": token_id(uuid.uuid4().hex), + "status": "approved", + "address": "127.0.0.1", + "ticket": token_id(uuid.uuid4().hex), + } + + +@pytest.fixture +async def sessions(people: People, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Sessions]: + store = Sessions(people) + monkeypatch.setattr(oauth, "store", store) + monkeypatch.setattr(settings, "jwt_secret", "unit-native-session-sealing-root") + monkeypatch.setattr(settings, "oauth_login_provider", "") + try: + yield store + finally: + for key in store.keys: + current = await claims.get(people.client, key) + if current is not None: + await claims._drop(people.client, key, current) + if store.families: + await people.client.execute( + "ALTER TABLE loom.revoked_tokens DELETE WHERE jti IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + params={"ids": list(store.families)}, + ) + + +async def test_real_refresh_round_trip_putback_and_rotation_preserve_email( + sessions: Sessions, +) -> None: + email = sessions.people.email("session") + grant = sessions.grant(email=email) + grant["upstream_refresh"] = "synthetic-upstream-before" + token = uuid.uuid4().hex + await sessions.save_refresh(token, grant) + row = await sessions.people.client.query_one( + "SELECT verified_email, upstream_refresh FROM loom.oauth_state FINAL " + "WHERE record_type = 'session_refresh' AND id = {id:String}", + params={"id": token_id(token)}, + ) + assert row is not None and row["verified_email"] == email + assert row["upstream_refresh"] != grant["upstream_refresh"] + assert ( + unseal(oauth_store.UPSTREAM_PURPOSE, row["upstream_refresh"]) == grant["upstream_refresh"] + ) + spent = await sessions.take_refresh(token, grant["client_id"]) + assert spent is not None and spent["verified_email"] == email + key = f"family:{grant['family']}" + held = spent["held"][key] + assert await claims.get(sessions.people.client, key) == held + spent["upstream_refresh"] = "synthetic-upstream-rotated" + await sessions.save_refresh(token, spent) + retried = await sessions.take_refresh(token, grant["client_id"]) + assert retried is not None and retried["family"] == grant["family"] + assert retried["verified_email"] == email + assert retried["upstream_refresh"] == "synthetic-upstream-rotated" + replacement = uuid.uuid4().hex + await sessions.save_refresh(replacement, {**retried, "generation": 1}) + rotated = await sessions.take_refresh(replacement, grant["client_id"]) + assert rotated is not None and rotated["generation"] == 1 + assert rotated["verified_email"] == email + assert rotated["family"] == grant["family"] + with pytest.raises(oauth_store.CodeReplayed): + await sessions.take_refresh(token, grant["client_id"]) + + +@pytest.mark.parametrize("kind", ["code", "device"]) +async def test_real_keeper_records_preserve_email_and_exact_held_cas( + sessions: Sessions, kind: str +) -> None: + email = sessions.people.email("keeper") + grant = sessions.grant(email=email) + credential = uuid.uuid4().hex + if kind == "code": + await sessions.save_code(credential, grant) + key = f"code:{token_id(credential)}" + spent = await sessions.take_code(credential, grant["client_id"], lambda _: True) + else: + device = sessions.make_device(grant) + device["id"] = token_id(credential) + await sessions.save_device(device) + key = f"device:{device['id']}" + spent = await sessions.take_device(credential) + assert spent is not None and spent["verified_email"] == email + held = spent["held"][key] + assert await claims.get(sessions.people.client, key) == held + assert json.loads(held)["verified_email"] == email + if kind == "code": + await sessions.save_code(credential, spent) # type: ignore[arg-type] + else: + await sessions.save_device(spent) # type: ignore[arg-type] + restored = await claims.get(sessions.people.client, key) + assert restored is not None and restored != held + assert json.loads(restored)["verified_email"] == email + assert json.loads(restored)["spent"] is False + with pytest.raises(claims.Changed): + await sessions._put(key, held, json.loads(held)) + + +@pytest.mark.parametrize("kind", ["code", "device"]) +async def test_previous_keeper_writer_drops_email_without_adding_proof( + sessions: Sessions, kind: str +) -> None: + email = sessions.people.email("previous") + grant = sessions.grant(email=email) + if kind == "code": + fields = previous._CODE_FIELDS + record = oauth_store._record(grant, fields, spent=False) + old_read = previous._code(record) + current_read = oauth_store._code + else: + device = sessions.make_device(grant) + fields = previous._DEVICE_FIELDS + record = oauth_store._record(device, fields, spent=False) + old_read = previous._device(record) + current_read = oauth_store._device + assert "verified_email" not in old_read + old_write = previous._record(old_read, fields, spent=False) + assert "verified_email" not in old_write + key = f"{kind}:{uuid.uuid4().hex}" + held = await sessions._put(key, None, record) + rewritten = await sessions._put(key, held, old_write) + assert await claims.get(sessions.people.client, key) == rewritten + parsed = current_read(json.loads(rewritten)) + assert parsed["verified_email"] == "" + assert parsed["subject"] == grant["subject"] + await sessions.people.provision(email) + assert await sessions.people.refused(parsed["subject"], verified_email=parsed["verified_email"]) + await sessions.people.account(grant["subject"], email) + assert not await sessions.people.refused( + parsed["subject"], verified_email=parsed["verified_email"] + ) + + +async def test_previous_refresh_writer_uses_default_empty_and_cannot_borrow_a_stored_email( + sessions: Sessions, +) -> None: + email = sessions.people.email("previous-refresh") + grant = sessions.grant(email=email) + token = uuid.uuid4().hex + await sessions.save_refresh(token, grant) + row = previous._refresh_row(token_id(token), grant, time.time_ns()) + assert "verified_email" not in row + row["version"] += 1 + await sessions.people.client.insert("loom.oauth_state", [row]) + loaded = await sessions.take_refresh(token, grant["client_id"]) + assert loaded is not None and loaded["verified_email"] == "" + assert loaded["family"] == grant["family"] + assert loaded["held"][f"family:{grant['family']}"] == await claims.get( + sessions.people.client, f"family:{grant['family']}" + ) + await sessions.people.provision(email) + assert await sessions.people.refused(loaded["subject"], verified_email=loaded["verified_email"]) + + +async def test_released_account_cannot_refresh_with_its_stored_verified_email( + sessions: Sessions, monkeypatch: pytest.MonkeyPatch +) -> None: + email = sessions.people.email("released") + grant = sessions.grant(email=email) + namespace = await sessions.people.account(grant["subject"], email) + await sessions.people.provision(email) + token = uuid.uuid4().hex + await sessions.save_refresh(token, grant) + assert not await sessions.people.refused(grant["subject"], verified_email=email) + released = await release_claim( + org=grant["org"], namespace=namespace, reason="test", actor_subject="host" + ) + assert released is not None and released.released + + async def forbidden_mint(*args: Any, **kwargs: Any) -> Any: + pytest.fail("a released account reached token issuance") + + monkeypatch.setattr(oauth, "issue_tokens", forbidden_mint) + with pytest.raises(oauth.OAuthError) as refused: + await oauth._refresh_token_grant({"client_id": grant["client_id"], "refresh_token": token}) + assert refused.value.status_code == 400 + with pytest.raises(oauth_store.SessionRetired): + await sessions.take_refresh(token, grant["client_id"]) + + +async def test_native_local_policy_outage_puts_back_the_same_refresh_without_rotation( + sessions: Sessions, monkeypatch: pytest.MonkeyPatch +) -> None: + email = sessions.people.email("outage") + grant = sessions.grant(email=email) + grant["upstream_refresh"] = "synthetic-upstream-current" + token = uuid.uuid4().hex + await sessions.save_refresh(token, grant) + await sessions.people.account(grant["subject"], email) + original = sessions.people.client.query_one + outage = OSError("synthetic local policy read failure") + + async def failed_policy(sql: str, *args: Any, **kwargs: Any) -> Any: + if "loom.namespaces" in sql: + raise outage + return await original(sql, *args, **kwargs) + + async def forbidden_upstream(*args: Any, **kwargs: Any) -> Any: + pytest.fail("local policy outage reached the identity provider") + + monkeypatch.setattr(sessions.people.client, "query_one", failed_policy) + monkeypatch.setattr(oauth, "_ask_identity_provider", forbidden_upstream) + monkeypatch.setattr(settings, "oauth_login_provider", "oidc") + with pytest.raises(oauth.OAuthError) as unavailable: + await oauth._refresh_token_grant({"client_id": grant["client_id"], "refresh_token": token}) + assert unavailable.value.status_code == 503 + assert isinstance(unavailable.value.__cause__, SignInUnavailable) + assert unavailable.value.__cause__.__cause__ is outage + retried = await sessions.take_refresh(token, grant["client_id"]) + assert retried is not None and retried["family"] == grant["family"] + assert retried["verified_email"] == email + assert retried["upstream_refresh"] == grant["upstream_refresh"] diff --git a/tests/unit/mcp_probe.py b/tests/unit/mcp_probe.py index d4ff03498c..f0e1881233 100644 --- a/tests/unit/mcp_probe.py +++ b/tests/unit/mcp_probe.py @@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -103,7 +104,7 @@ def _verify(token: str) -> AuthInfo: # Every served op has a latency budget; these two are not in the registry. monkeypatch.setitem(OP_BUDGETS_P99_MS, op.name, 10_000) monkeypatch.setitem(OP_BUDGETS_P99_MS, flat.name, 10_000) - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr(http_mod, "_emit_event", lambda *a, **k: None) monkeypatch.setattr(http_mod, "_record_failure", lambda *a, **k: None) app = FastAPI() diff --git a/tests/unit/test_admission_gates.py b/tests/unit/test_admission_gates.py index 2dae4cbce1..302d7e6ee3 100644 --- a/tests/unit/test_admission_gates.py +++ b/tests/unit/test_admission_gates.py @@ -20,6 +20,7 @@ import dataclasses import logging from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -154,7 +155,7 @@ def _verify(token: str) -> AuthInfo: # HTTP monkeypatch.setattr(http_mod, "ALL_OPERATIONS", [op]) monkeypatch.setitem(OP_BUDGETS_P99_MS, op.name, 10_000) # a probe op's budget - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr( http_mod, "_emit_event", lambda *a, **k: seen["ledger"].append(("http", k["error_code"])) ) diff --git a/tests/unit/test_admission_parity.py b/tests/unit/test_admission_parity.py index c09f4099c7..931cca1483 100644 --- a/tests/unit/test_admission_parity.py +++ b/tests/unit/test_admission_parity.py @@ -10,6 +10,7 @@ import asyncio from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -119,7 +120,7 @@ def _verify(token: str) -> AuthInfo: raise TokenError("bad token") return _principal() - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) app = FastAPI() app.include_router(http_mod.build_router()) client = TestClient(app) diff --git a/tests/unit/test_assistant_bridge.py b/tests/unit/test_assistant_bridge.py index fd549e023d..b606e9f96b 100644 --- a/tests/unit/test_assistant_bridge.py +++ b/tests/unit/test_assistant_bridge.py @@ -9,6 +9,7 @@ import json import time from typing import Any +from unittest.mock import AsyncMock import httpx import pytest @@ -441,7 +442,9 @@ async def classified(_auth: Any, org: str, namespace: str, _args: Any) -> tuple[ limiter = ratelimit.Limiter() monkeypatch.setattr(ratelimit, "get_limiter", lambda: limiter) - monkeypatch.setattr(routes, "verify_bearer_header", lambda *_a, **_k: _caller()) + monkeypatch.setattr( + routes, "verify_bearer_header", AsyncMock(side_effect=lambda *_a, **_k: _caller()) + ) monkeypatch.setattr(routes, "check_acted_for", nothing) monkeypatch.setattr(routes, "note_restricted_access", nothing) monkeypatch.setattr(routes, "emit_agent_event", emit) @@ -687,7 +690,9 @@ def test_the_answers_row_names_who_acted_and_its_budget( http, rows = client monkeypatch.setattr( - routes, "verify_bearer_header", lambda *_a, **_k: _caller(impersonated_by="trusted-ui") + routes, + "verify_bearer_header", + AsyncMock(side_effect=lambda *_a, **_k: _caller(impersonated_by="trusted-ui")), ) http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION}) (row,) = rows @@ -957,7 +962,9 @@ def test_a_refusal_after_the_token_is_read_carries_its_class_claim( http, rows = client monkeypatch.setattr( - routes, "verify_bearer_header", lambda *_a, **_k: _caller(traffic_class="bench") + routes, + "verify_bearer_header", + AsyncMock(side_effect=lambda *_a, **_k: _caller(traffic_class="bench")), ) world.denied = True # no viewer: refused after the claim is bound, before classifying r = http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION}) @@ -1099,7 +1106,9 @@ def test_an_org_wide_token_is_refused_with_a_row_naming_no_org( http, rows = client monkeypatch.setattr( - routes, "verify_bearer_header", lambda *_a, **_k: _caller(org="*", namespaces=("*",)) + routes, + "verify_bearer_header", + AsyncMock(side_effect=lambda *_a, **_k: _caller(org="*", namespaces=("*",))), ) r = http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION}) assert r.status_code == 403 and r.json()["detail"]["code"] == "AUTH_INSUFFICIENT" @@ -1236,7 +1245,9 @@ def test_an_answer_token_or_its_child_cannot_ask_the_chat_route( ) -> None: from server.assistant.app import routes - monkeypatch.setattr(routes, "verify_bearer_header", lambda *_a, **_k: _caller(**token)) + monkeypatch.setattr( + routes, "verify_bearer_header", AsyncMock(side_effect=lambda *_a, **_k: _caller(**token)) + ) http, rows = client r = http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION}) assert r.status_code == 403 @@ -1306,7 +1317,9 @@ def test_an_answer_token_acting_for_someone_is_still_refused( # A trusted caller's answer token naming another subject keeps its consumer. bearer = _caller(subject="trusted-ui", consumer=ASSISTANT_CONSUMER) - monkeypatch.setattr(routes, "verify_bearer_header", lambda *_a, **_k: bearer) + monkeypatch.setattr( + routes, "verify_bearer_header", AsyncMock(side_effect=lambda *_a, **_k: bearer) + ) monkeypatch.setattr(settings, "trusted_callers", "trusted-ui") http, rows = client headers = {**HEADERS, "X-Loom-On-Behalf-Of": "bob"} @@ -1350,7 +1363,9 @@ def test_a_process_set_up_with_auth_off_is_not_ready(monkeypatch: pytest.MonkeyP def test_the_chat_route_with_auth_off_says_so(client: Any, monkeypatch: pytest.MonkeyPatch) -> None: from server.assistant.app import routes - monkeypatch.setattr(routes, "verify_bearer_header", lambda *_a, **_k: None) + monkeypatch.setattr( + routes, "verify_bearer_header", AsyncMock(side_effect=lambda *_a, **_k: None) + ) monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: False) http, _rows = client r = http.post(ROUTE, headers={"X-Loom-Namespace": NS}, json={"messages": QUESTION}) diff --git a/tests/unit/test_auth_routes.py b/tests/unit/test_auth_routes.py index 6eaa432990..73df27f29b 100644 --- a/tests/unit/test_auth_routes.py +++ b/tests/unit/test_auth_routes.py @@ -5,6 +5,7 @@ import json from datetime import datetime +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -102,7 +103,7 @@ def test_whoami_while_revocation_is_unloaded_is_a_retryable_503( def _unavailable(_token: str) -> None: raise RevocationUnavailable("revocation list not loaded") - monkeypatch.setattr(routes, "verify_token", _unavailable) + monkeypatch.setattr(routes, "authenticate_token", AsyncMock(side_effect=_unavailable)) r = _client().get("/v1/whoami", headers={"Authorization": "Bearer t"}) assert r.status_code == 503 assert r.headers["Retry-After"] == "5" diff --git a/tests/unit/test_bearer_parse.py b/tests/unit/test_bearer_parse.py index 0ff589ed06..63739510ef 100644 --- a/tests/unit/test_bearer_parse.py +++ b/tests/unit/test_bearer_parse.py @@ -17,9 +17,9 @@ @pytest.mark.parametrize("header", [None, "", " "]) -def test_a_blank_header_is_no_credential(header: str | None) -> None: +async def test_a_blank_header_is_no_credential(header: str | None) -> None: assert bearer_token(header) is None - assert verify_bearer_header(header) is None + assert (await verify_bearer_header(header)) is None @pytest.mark.parametrize("header", ["Bearer abc", "bearer abc", "BEARER abc ", " Bearer abc"]) @@ -30,11 +30,11 @@ def test_the_scheme_is_case_insensitive_and_the_token_trimmed(header: str) -> No @pytest.mark.parametrize( "header", ["Basic dXNlcjpwYXNz", "Bearer", "Bearer ", "Token abc", "abc"] ) -def test_anything_else_is_a_credential_loom_cannot_read(header: str) -> None: +async def test_anything_else_is_a_credential_loom_cannot_read(header: str) -> None: with pytest.raises(TokenError): bearer_token(header) with pytest.raises(HTTPException) as e: - verify_bearer_header(header) + (await verify_bearer_header(header)) assert e.value.status_code == 401 and e.value.detail["code"] == "AUTH_INVALID" diff --git a/tests/unit/test_ch_caller_shaped.py b/tests/unit/test_ch_caller_shaped.py index 629af04915..83ce08f293 100644 --- a/tests/unit/test_ch_caller_shaped.py +++ b/tests/unit/test_ch_caller_shaped.py @@ -153,7 +153,7 @@ def test_a_route_outside_the_registry_carries_the_quota_key_once_the_bearer_veri @app.get("/probe") async def probe(request: Request) -> dict[str, Any]: - verify_bearer_header(request.headers.get("Authorization")) + (await verify_bearer_header(request.headers.get("Authorization"))) return {"key": ch.statement_settings(_SELECT).get("quota_key")} tok = mint_token(subject="alice", org="acme", namespaces=["n"], roles=["viewer"]) diff --git a/tests/unit/test_cross_org_rule.py b/tests/unit/test_cross_org_rule.py index 9d24140a7a..128be16d5d 100644 --- a/tests/unit/test_cross_org_rule.py +++ b/tests/unit/test_cross_org_rule.py @@ -11,6 +11,7 @@ import asyncio from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -53,7 +54,7 @@ class _Out(BaseModel): ok: bool = True -def _verify(token: str) -> AuthInfo: +def _verify(token: str, *, secret: str | None = None) -> AuthInfo: if token not in TOKENS: raise TokenError("bad token") return TOKENS[token] @@ -94,7 +95,7 @@ async def _no_defaults(auth: Any, client_name: str | None = None) -> None: monkeypatch.setattr(settings, "mcp_auth_mode", "required") monkeypatch.setattr(settings, "oauth_enabled", False) monkeypatch.setattr(settings, "identity_provider", "", raising=False) - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr("server.auth.jwt.verify_token", _verify) monkeypatch.setattr(http_mod, "ALL_OPERATIONS", [op]) monkeypatch.setitem(OP_BUDGETS_P99_MS, op.name, 10_000) # a probe op's budget diff --git a/tests/unit/test_delegation.py b/tests/unit/test_delegation.py index a0f89219c3..98624bd7f4 100644 --- a/tests/unit/test_delegation.py +++ b/tests/unit/test_delegation.py @@ -899,10 +899,10 @@ async def test_a_primary_scoped_token_writes_by_its_role_map(world) -> None: assert not await _reaches(bound, "a", Role.WRITER) -def test_the_dev_mode_parent_is_a_flat_writer() -> None: +async def test_the_dev_mode_parent_is_a_flat_writer() -> None: from server.identity.app import me_service_accounts - parent = me_service_accounts.dev_parent( + parent = await me_service_accounts.dev_parent( SimpleNamespace(headers={}), {"subject": "d", "org": "g"} ) assert (parent.subject, parent.org, parent.roles, parent.creator) == ( diff --git a/tests/unit/test_dev_mode_token_owner.py b/tests/unit/test_dev_mode_token_owner.py index d7361c2841..5fa8877e2c 100644 --- a/tests/unit/test_dev_mode_token_owner.py +++ b/tests/unit/test_dev_mode_token_owner.py @@ -1,13 +1,14 @@ """With auth off (single-user included), every route that saves a token and the -token list that shows and revokes it agree on who the caller is: a token minted -by `/v1/admin/token.mint` or the Setup page is listed on the Tokens page and -revoked from it, whether or not a bearer is sent.""" +token list that shows and revokes it agree on who the caller is: a valid bearer +names its owner, and an absent bearer selects the deliberate dev identity. +An invalid supplied credential is refused before anything is minted or saved.""" from __future__ import annotations import time from datetime import datetime, timedelta from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -25,8 +26,10 @@ class _Store: def __init__(self) -> None: self.rows: dict[tuple[str, str], dict[str, Any]] = {} self.revocations: list[dict[str, Any]] = [] # loom.revoked_tokens + self.writes: list[str] = [] async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: + self.writes.append(table) if table == "loom.revoked_tokens": self.revocations.extend(rows) if table == "loom.service_accounts": @@ -58,7 +61,7 @@ def client(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> T from server.insights.app import admin_routes if request.param == "single-user": - # Single-user overrides auth_required=True: no bearer is checked. + # Single-user retains its deliberate identity when no bearer is supplied. monkeypatch.setattr(settings, "auth_required", True) monkeypatch.setattr(settings, "identity_provider", "single-user") else: @@ -122,6 +125,7 @@ async def _no_pick(*_a: Any, **_k: Any) -> str: monkeypatch.setattr(token_grants, "revoke_token_grants", _zero) app = FastAPI() + app.state.token_store = store app.include_router(admin_routes.org_admin_router) app.include_router(me_service_accounts.router) return TestClient(app) @@ -134,8 +138,32 @@ def _alice() -> dict[str, str]: return {"Authorization": f"Bearer {token}"} -# Auth off ignores a bearer that does not verify: the caller is as if none were sent. -NO_CALLER = [{}, {"Authorization": "Bearer not-a-jwt"}, {"Authorization": "Basic eA=="}] +AUTH_HEADERS = [ + pytest.param({}, id="absent"), + pytest.param({"Authorization": "Bearer not-a-jwt"}, id="bad-bearer"), + pytest.param({"Authorization": "Basic eA=="}, id="basic"), +] + + +def _refused_without_writes( + client: TestClient, + headers: dict[str, str], + path: str, + body: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.auth.delegation import Delegated + + def _no_mint(*args: Any, **kwargs: Any) -> str: + pytest.fail("an invalid supplied credential reached token minting") + + monkeypatch.setattr(Delegated, "mint", _no_mint) + response = client.post(path, json=body, headers=headers) + assert response.status_code == 401, response.text + assert response.json()["detail"]["code"] == "AUTH_INVALID" + store = client.app.state.token_store + assert store.writes == [] + assert store.rows == {} and store.revocations == [] def _listed(client: TestClient, headers: dict[str, str]) -> dict[str, dict[str, Any]]: @@ -164,16 +192,30 @@ def _mint_default_token(client: TestClient, headers: dict[str, str]) -> str: return name -@pytest.mark.parametrize("headers", NO_CALLER) +@pytest.mark.parametrize("headers", AUTH_HEADERS) def test_a_token_minted_by_token_mint_is_listed_and_revoked( - client: TestClient, headers: dict[str, str] + client: TestClient, headers: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: + if headers: + _refused_without_writes( + client, + headers, + "/v1/admin/token.mint", + { + "name": "ci-bot", + "org": settings.self_signup_org, + "namespaces": ["home"], + "roles": ["viewer"], + }, + monkeypatch, + ) + return _token_mint(client, headers, settings.self_signup_org) assert _listed(client, headers)["ci-bot"]["created_by"] == "you" _revoke_and_check(client, headers, "ci-bot") -@pytest.mark.parametrize("headers", NO_CALLER) +@pytest.mark.parametrize("headers", AUTH_HEADERS) def test_the_single_user_mints_on_the_namespace_it_administers( client: TestClient, headers: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: @@ -184,12 +226,17 @@ def test_the_single_user_mints_on_the_namespace_it_administers( org = settings.self_signup_org ns = settings.default_namespace + body = {"name": "laptop", "namespaces": [ns, "*"], "ttl_days": 90} + if headers: + _refused_without_writes( + client, headers, "/v1/me/service_accounts.create", body, monkeypatch + ) + return async def _bootstrap_grant(o: str, n: str, subject: str) -> Role | None: return Role.ADMIN if (o, n, subject) == (org, ns, "you") else None monkeypatch.setattr(authz, "grant_role_for", _bootstrap_grant) - body = {"name": "laptop", "namespaces": [ns, "*"], "ttl_days": 90} r = client.post("/v1/me/service_accounts.create", json=body, headers=headers) assert r.status_code == 200, r.text assert r.json()["org"] == org @@ -212,10 +259,13 @@ def test_a_token_the_old_owner_made_is_listed_and_revoked_by_name(client: TestCl assert rows[0]["revoked_at"] is not None -@pytest.mark.parametrize("headers", NO_CALLER) +@pytest.mark.parametrize("headers", AUTH_HEADERS) def test_a_token_made_without_a_name_is_listed_and_revoked( - client: TestClient, headers: dict[str, str] + client: TestClient, headers: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> None: + if headers: + _refused_without_writes(client, headers, "/v1/me/service_accounts.create", {}, monkeypatch) + return _revoke_and_check(client, headers, _mint_default_token(client, headers)) @@ -344,7 +394,7 @@ def _unavailable(token: str) -> Any: raise jwt_mod.RevocationUnavailable("revocation list not loaded") monkeypatch.setattr(jwt_mod, "verify_token", _unavailable) - monkeypatch.setattr(bearer, "verify_token", _unavailable) + monkeypatch.setattr(bearer, "authenticate_token", AsyncMock(side_effect=_unavailable)) r = _create(client, _alice(), namespaces=["home"]) assert r.status_code == 503, r.text assert r.headers["Retry-After"] == "5" diff --git a/tests/unit/test_http_idempotency_after_auth.py b/tests/unit/test_http_idempotency_after_auth.py index 7f5730ec70..b4b2c11c99 100644 --- a/tests/unit/test_http_idempotency_after_auth.py +++ b/tests/unit/test_http_idempotency_after_auth.py @@ -10,6 +10,7 @@ import dataclasses from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -71,7 +72,7 @@ async def _store(*, idempotency_key: str, body_hash: str, response_body: Any, ** monkeypatch.setattr(http_mod, "ALL_OPERATIONS", [op]) monkeypatch.setattr(settings, "auth_required", True) monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) async def _admit(*_a: Any, **_k: Any) -> gates.Target: return gates.Target(org="o", namespace="n", named_namespace="n", register_namespace="n") diff --git a/tests/unit/test_idempotency_across_transports.py b/tests/unit/test_idempotency_across_transports.py index c4ad69752f..93605977d4 100644 --- a/tests/unit/test_idempotency_across_transports.py +++ b/tests/unit/test_idempotency_across_transports.py @@ -10,6 +10,7 @@ import asyncio from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -108,7 +109,7 @@ async def _no_defaults(auth: Any, client_name: str | None = None) -> None: monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) monkeypatch.setattr(http_mod, "ALL_OPERATIONS", [op]) monkeypatch.setitem(OP_BUDGETS_P99_MS, op.name, 10_000) # a probe op's budget - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr(http_mod, "_emit_event", lambda *a, **k: None) monkeypatch.setitem(mcp_call.TOOLS_BY_NAME, TOOL, op) monkeypatch.setattr(mcp_call, "_caller_defaults", _no_defaults) diff --git a/tests/unit/test_op_transport_contract.py b/tests/unit/test_op_transport_contract.py index 6f55255285..0c7d51fce3 100644 --- a/tests/unit/test_op_transport_contract.py +++ b/tests/unit/test_op_transport_contract.py @@ -37,6 +37,7 @@ import typing from collections.abc import Iterator from typing import Any +from unittest.mock import AsyncMock import annotated_types import pytest @@ -155,8 +156,12 @@ def _principal(role: Role) -> AuthInfo: # holds it there. _ORG_CHECKED_IN_HANDLER = { "erasure.status": "test_transport_security::test_erasure_status_foreign_org_token_is_not_found", - "memory.cross_promote": "test_cross_promote_source_scope::test_another_orgs_admin_cannot_cross_promote", - "memory.copy_to": "test_cross_promote_source_scope::test_another_orgs_admin_cannot_copy_into_or_out_of_an_org", + "memory.cross_promote": ( + "test_cross_promote_source_scope::test_another_orgs_admin_cannot_cross_promote" + ), + "memory.copy_to": ( + "test_cross_promote_source_scope::test_another_orgs_admin_cannot_copy_into_or_out_of_an_org" + ), } @@ -220,7 +225,7 @@ async def _none(*_a: Any, **_k: Any) -> None: monkeypatch.setattr(settings, "tracker_org", ORG) monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) monkeypatch.setattr(http_mod, "ALL_OPERATIONS", ops) - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr(http_mod, "_emit_event", lambda *a, **k: None) for op in ops: monkeypatch.setitem(mcp_call.TOOLS_BY_NAME, op.mcp_tool_name, op) diff --git a/tests/unit/test_ratelimit_per_org_namespace.py b/tests/unit/test_ratelimit_per_org_namespace.py index 7fd3cc9efb..53865c23d0 100644 --- a/tests/unit/test_ratelimit_per_org_namespace.py +++ b/tests/unit/test_ratelimit_per_org_namespace.py @@ -12,6 +12,7 @@ import dataclasses from pathlib import Path from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI, HTTPException @@ -74,7 +75,7 @@ async def _noop(*_a: Any, **_k: Any) -> None: monkeypatch.setattr(http_mod, "ALL_OPERATIONS", [op]) monkeypatch.setattr(settings, "auth_required", True) monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) - monkeypatch.setattr(bearer_mod, "verify_token", _verify) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_verify)) monkeypatch.setattr(http_mod, "_authorize", _authorize) import server.core.ratelimit as ratelimit_mod diff --git a/tests/unit/test_raw_routes_restricted_audit.py b/tests/unit/test_raw_routes_restricted_audit.py index 53c5385b14..6d5db0a573 100644 --- a/tests/unit/test_raw_routes_restricted_audit.py +++ b/tests/unit/test_raw_routes_restricted_audit.py @@ -21,6 +21,7 @@ from collections.abc import Iterator from pathlib import Path from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -66,7 +67,9 @@ async def _emit(event_type: str, **kw: Any) -> str: is_service_account=False, token_id="tok-alice", ) - monkeypatch.setattr("server.auth.bearer.verify_token", lambda token: alice) + monkeypatch.setattr( + "server.auth.bearer.authenticate_token", AsyncMock(side_effect=lambda token: alice) + ) yield rows diff --git a/tests/unit/test_request_sli.py b/tests/unit/test_request_sli.py index f4710e049c..f0a4573e5a 100644 --- a/tests/unit/test_request_sli.py +++ b/tests/unit/test_request_sli.py @@ -7,6 +7,7 @@ import contextvars import re from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException @@ -217,7 +218,7 @@ def add(self, n: int, attributes: dict[str, str]) -> None: self.calls.append(attributes) -def test_a_refused_bearer_is_counted_once_on_each_transport(monkeypatch) -> None: +async def test_a_refused_bearer_is_counted_once_on_each_transport(monkeypatch) -> None: from server.auth import bearer from server.observability import metrics from server.transport import mcp_auth @@ -225,10 +226,10 @@ def test_a_refused_bearer_is_counted_once_on_each_transport(monkeypatch) -> None counter = _Counter() monkeypatch.setattr(metrics, "_auth_refusals", counter) with pytest.raises(HTTPException) as e: - bearer.verify_bearer_header("Basic abc") + (await bearer.verify_bearer_header("Basic abc")) assert e.value.status_code == 401 with pytest.raises(HTTPException): - bearer.verify_bearer("not-a-jwt") + (await bearer.verify_bearer("not-a-jwt")) assert counter.calls == [ {"reason": "malformed", "loom.transport": "http"}, {"reason": "invalid", "loom.transport": "http"}, @@ -247,7 +248,7 @@ class _Unchecked: for check in (lambda: bearer.verify_bearer(token), lambda: dev_bearer(_Unchecked())): with pytest.raises(HTTPException) as e: - check() + await check() assert e.value.status_code == 503 assert counter.calls == [{"reason": "unavailable", "loom.transport": "http"}] * 2 @@ -269,7 +270,7 @@ class _Req: ] -def test_a_missing_bearer_is_counted_on_every_http_route(monkeypatch) -> None: +async def test_a_missing_bearer_is_counted_on_every_http_route(monkeypatch) -> None: from starlette.requests import Request from server.auth import routes @@ -289,19 +290,19 @@ def test_a_missing_bearer_is_counted_on_every_http_route(monkeypatch) -> None: me_callers._authed, watch_routes._bearer_auth, lambda r: admin_routes._verified_admin(r, Role.ADMIN), - lambda r: asyncio.run(download_routes.knowledge_download(r, "o", "n", "s")), + lambda r: download_routes.knowledge_download(r, "o", "n", "s"), ) for refuse in refusals: with pytest.raises(HTTPException) as e: - refuse(request) + await refuse(request) assert e.value.status_code == 401 assert counter.calls == [{"reason": "missing", "loom.transport": "http"}] * len(refusals) # The token check counts a refused token, not a signed-out caller. counter.calls.clear() - assert asyncio.run(routes.whoami(request)) == {"authenticated": False} + assert await routes.whoami(request) == {"authenticated": False} bad = Request({"type": "http", "headers": [(b"authorization", b"Bearer not-a-jwt")]}) - assert asyncio.run(routes.whoami(bad))["authenticated"] is False + assert (await routes.whoami(bad))["authenticated"] is False assert counter.calls == [{"reason": "invalid", "loom.transport": "http"}] @@ -1663,7 +1664,7 @@ async def _noop(*a: Any, **k: Any) -> None: subject="eve", org="eve-org", namespaces=("*",), - roles=(Role.VIEWER,), + roles=(Role.PLATFORM_ADMIN,), is_service_account=True, ) for arguments in ({}, {"org": "", "namespace": ""}): @@ -1683,7 +1684,7 @@ async def _refuse(*a: Any, **k: Any) -> None: # HTTP with a token, through the endpoint: refused while its body is # validated (a null org) and at the gate, naming no org or an empty one. rows.clear() - monkeypatch.setattr(http_mod, "_extract_auth", lambda request: eve) + monkeypatch.setattr(http_mod, "_extract_auth", AsyncMock(side_effect=lambda request: eve)) client = TestClient(app) assert client.post("/v1/code.repos", json={"org": None}).status_code == 422 for body in ({}, {"org": "", "namespace": ""}, {"org": " "}, {"org": "-", "namespace": "n"}): diff --git a/tests/unit/test_revocation_fails_closed_until_loaded.py b/tests/unit/test_revocation_fails_closed_until_loaded.py index 2124ddabe3..3b1c80d5d6 100644 --- a/tests/unit/test_revocation_fails_closed_until_loaded.py +++ b/tests/unit/test_revocation_fails_closed_until_loaded.py @@ -11,6 +11,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import AsyncMock import pytest @@ -87,7 +88,7 @@ async def test_a_snapshot_past_the_stale_bound_refuses_until_a_refresh_lands( assert verify_token(_token(), secret=SECRET).subject == "alice" -def test_http_answers_an_unloaded_list_with_a_retryable_503( +async def test_http_answers_an_unloaded_list_with_a_retryable_503( monkeypatch: pytest.MonkeyPatch, ) -> None: from fastapi import HTTPException @@ -97,19 +98,19 @@ def test_http_answers_an_unloaded_list_with_a_retryable_503( def _unavailable(token: str) -> Any: raise RevocationUnavailable("token revocation list not loaded yet; retry shortly") - monkeypatch.setattr(bearer_mod, "verify_token", _unavailable) + monkeypatch.setattr(bearer_mod, "authenticate_token", AsyncMock(side_effect=_unavailable)) class _Req: headers = {"Authorization": "Bearer x.y.z"} with pytest.raises(HTTPException) as err: - http_mod._extract_auth(_Req()) # type: ignore[arg-type] + (await http_mod._extract_auth(_Req())) # type: ignore[arg-type] assert err.value.status_code == 503 assert err.value.detail["code"] == "AUTH_UNAVAILABLE" assert err.value.headers == {"Retry-After": "5"} -def test_every_http_bearer_check_answers_an_unloaded_list_with_503( +async def test_every_http_bearer_check_answers_an_unloaded_list_with_503( monkeypatch: pytest.MonkeyPatch, ) -> None: """The admin router, claim release and the knowledge download share one check.""" @@ -120,14 +121,14 @@ def test_every_http_bearer_check_answers_an_unloaded_list_with_503( monkeypatch.setattr(rev, "_loaded", False) tok = mint_token(subject="s", org="o", namespaces=["n"], roles=["admin"], ttl_seconds=60) with pytest.raises(HTTPException) as err: - verify_bearer(tok) + await verify_bearer(tok) assert err.value.status_code == 503 and err.value.headers == {"Retry-After": "5"} with pytest.raises(HTTPException) as bad: - verify_bearer("garbage") + await verify_bearer("garbage") assert bad.value.status_code == 401 -def test_me_and_task_watch_answer_an_unloaded_list_with_503( +async def test_me_and_task_watch_answer_an_unloaded_list_with_503( monkeypatch: pytest.MonkeyPatch, ) -> None: """The caller API's bearer checks and the task watch stream use the same check: a @@ -151,12 +152,12 @@ def test_me_and_task_watch_answer_an_unloaded_list_with_503( ] for check in checks: with pytest.raises(HTTPException) as err: - check(req) + await check(req) assert err.value.status_code == 503, check.__name__ assert err.value.headers == {"Retry-After": "5"} monkeypatch.setattr(rev, "_loaded", True) bad: Any = SimpleNamespace(headers={"Authorization": "Bearer garbage"}, state=SimpleNamespace()) for check in checks: with pytest.raises(HTTPException) as err: - check(bad) + await check(bad) assert err.value.status_code == 401, check.__name__ diff --git a/tests/unit/test_scim_admission.py b/tests/unit/test_scim_admission.py new file mode 100644 index 0000000000..497fe038db --- /dev/null +++ b/tests/unit/test_scim_admission.py @@ -0,0 +1,370 @@ +"""Actual JWT and registered route admissions enforce the complete person graph.""" + +from __future__ import annotations + +import ast +import time +from pathlib import Path + +import pytest +from fastapi import FastAPI, HTTPException, Request +from fastapi.testclient import TestClient + +from server.auth import authz, bearer, jwt, keyring, revocation, routes, scim, tenancy +from server.core import cell +from server.identity.app import me_callers, me_routes, me_service_accounts +from server.operations.base import Role +from server.settings import settings +from tests.unit.test_scim_people import people as people + +SECRET = "unit-person-provisioning-signature-" * 3 + + +@pytest.fixture +def token(people, monkeypatch): + monkeypatch.setattr(cell, "gate_reason", lambda: "") + monkeypatch.setattr(cell, "first_identity", lambda: (True, "first")) + monkeypatch.setattr(keyring, "loaded", lambda: True) + monkeypatch.setattr(keyring, "stale", lambda: False) + monkeypatch.setattr(keyring, "verification_keys", lambda *_: [SECRET]) + monkeypatch.setattr(revocation, "loaded", lambda: True) + monkeypatch.setattr(revocation, "snapshot_age_seconds", lambda: 0) + monkeypatch.setattr(revocation, "is_revoked", lambda *_, **__: False) + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: True) + monkeypatch.setattr(settings, "self_signup_org", "acme") + + def mint(subject="alice@example.test", *, org="acme", parents=(), creator=None, sa=False): + return jwt.mint_token( + subject, + org, + ["*"], + ["platform_admin"], + secret=SECRET, + is_service_account=sa, + parent_subjects=list(parents), + parent_iats=[int(time.time())] * len(parents), + creator=creator, + ttl_seconds=60, + ) + + return mint + + +def request_for(raw): + return Request({"type": "http", "headers": [(b"authorization", f"Bearer {raw}".encode())]}) + + +async def test_raw_crypto_remains_independent_but_admission_requires_positive_provisioning(token): + raw = token() + assert jwt.verify_token(raw, secret=SECRET).subject == "alice@example.test" + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(raw, secret=SECRET) + + +@pytest.mark.parametrize("sa", [False, True]) +async def test_a_service_account_claim_cannot_exempt_a_person(token, sa): + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(token(sa=sa), secret=SECRET) + + +@pytest.mark.parametrize("field", ["creator", "parents"]) +async def test_creator_and_every_parent_belong_to_one_admission(people, token, field): + people.provision("alice@example.test") + kwargs = {field: ("missing@example.test",) if field == "parents" else "missing@example.test"} + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(token(**kwargs), secret=SECRET) + + +async def test_signed_gateway_root_overflow_refuses_before_any_identity_read(people, token): + people.provision("alice@example.test") + raw = token(parents=("alice@example.test",) * 1024) + assert len(jwt.verify_token(raw, secret=SECRET).parent_subjects) == 1024 + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(raw, secret=SECRET) + assert not people.reads + + +async def test_exact_root_limit_passes_after_counting_duplicates(people, token): + people.provision("alice@example.test") + admitted = await jwt.authenticate_token( + token(parents=("alice@example.test",) * 1023), secret=SECRET + ) + assert admitted.subject == "alice@example.test" + + +@pytest.mark.parametrize("active,refused", [(True, False), (False, True)]) +async def test_only_a_registered_active_bot_has_the_closed_exemption( + people, token, active, refused +): + people.bots["robot"] = active + if refused: + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(token("sa:robot:agent"), secret=SECRET) + else: + assert ( + await jwt.authenticate_token(token("sa:robot:agent"), secret=SECRET) + ).subject == "sa:robot:agent" + + +async def test_a_real_personal_service_account_uses_its_creator(people, token): + people.provision("alice@example.test") + people.services["sa:personal:one"] = {"created_by": "alice@example.test", "lineage": []} + assert ( + await jwt.authenticate_token(token("sa:personal:one"), secret=SECRET) + ).subject == "sa:personal:one" + + +async def test_admission_outage_is_retryable_auth_unavailability(people, token): + people.unavailable = True + with pytest.raises(jwt.AuthUnavailable): + await jwt.authenticate_token(token(), secret=SECRET) + + +def test_registered_whoami_refuses_an_unprovisioned_person(token): + app = FastAPI() + routes.attach(app) + response = TestClient(app).get("/v1/whoami", headers={"Authorization": f"Bearer {token()}"}) + assert response.status_code == 200 + assert response.json()["authenticated"] is False + + +def test_registered_whoami_reports_an_outage_as_retryable(people, token): + people.unavailable = True + app = FastAPI() + routes.attach(app) + response = TestClient(app).get("/v1/whoami", headers={"Authorization": f"Bearer {token()}"}) + assert response.status_code == 503 + assert response.headers["Retry-After"] == "5" + + +@pytest.mark.parametrize("auth_required", [False, True]) +def test_real_nonop_caller_route_refuses_before_writing(token, monkeypatch, auth_required): + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: auth_required) + + async def no_write(*_, **__): + pytest.fail("an unchecked person must not write the default namespace") + + from server.core.ch import get_client + + monkeypatch.setattr(get_client(), "insert", no_write, raising=False) + app = FastAPI() + me_routes.attach(app) + response = TestClient(app).post( + "/v1/me/set-default-namespace", + json={"namespace": "private"}, + headers={"Authorization": f"Bearer {token()}"}, + ) + assert response.status_code == 401 + + +@pytest.mark.parametrize("which", ["caller", "service_account"]) +async def test_actual_bearer_refusal_cannot_become_a_dev_identity(token, monkeypatch, which): + monkeypatch.setattr(type(settings), "auth_actually_required", lambda self: False) + with pytest.raises(HTTPException) as refused: + if which == "caller": + await me_callers.caller_identity(request_for(token())) + else: + await me_service_accounts.dev_caller(request_for(token()), {}) + assert refused.value.status_code == 401 + + +async def test_concrete_authorization_rechecks_the_full_wildcard_graph(people, token, monkeypatch): + people.provision("alice@example.test") + auth = await jwt.authenticate_token(token(org="*"), secret=SECRET) + people.accounts.append(("acme", "alice@example.test", "missing@example.test", False)) + + async def no_placement(*_, **__): + pytest.fail("provisioning refusal must precede tenancy placement") + + monkeypatch.setattr(tenancy, "ensure_held", no_placement) + with pytest.raises(PermissionError): + await authz.check_authorized(auth, Role.VIEWER, "acme", "private") + + +async def test_acting_for_a_person_retains_the_real_bearer_root(people, token, monkeypatch): + from server.auth.gateway import effective_auth + + for email in ("alice@example.test", "target@example.test"): + people.provision(email) + auth = await jwt.authenticate_token(token(org="*"), secret=SECRET) + monkeypatch.setattr(settings, "trusted_callers", "alice@example.test") + effective = effective_auth(auth, "target@example.test") + people.accounts.append(("acme", "alice@example.test", "missing@example.test", False)) + with pytest.raises(authz.ActedForRefused): + await authz.check_acted_for(effective, "acme") + + +async def test_bearer_header_chain_maps_missing_provisioning_to_401(token): + with pytest.raises(HTTPException) as refused: + await bearer.verify_bearer_header(f"Bearer {token()}") + assert refused.value.status_code == 401 + + +@pytest.mark.parametrize("outage,status", [(False, 401), (True, 503)]) +def test_registered_scim_groups_jwt_fallback_enforces_people( + people, token, monkeypatch, outage, status +): + from server.identity.app import scim_groups + + admitted = [] + real_decision = scim.refuses_subject + + async def decision(*args, **kwargs): + admitted.append((args, kwargs)) + return await real_decision(*args, **kwargs) + + real_query = people.query_one + + async def query(sql, params=None): + if "loom.principal_groups" in sql: + pytest.fail("a refused person must not read groups") + return await real_query(sql, params=params) + + monkeypatch.setattr(scim, "refuses_subject", decision) + monkeypatch.setattr(people, "query_one", query) + people.unavailable = outage + app = FastAPI() + scim_groups.attach(app) + response = TestClient(app).get( + "/scim/v2/Groups", headers={"Authorization": f"Bearer {token()}"} + ) + assert response.status_code == status + assert len(admitted) == 1 + assert admitted[0][0] == ("alice@example.test",) + + +@pytest.mark.parametrize("method", ["post", "delete"]) +@pytest.mark.parametrize("mode", ["required", "open"]) +@pytest.mark.parametrize("outage,status", [(False, 401), (True, 503)]) +def test_registered_mcp_cannot_downgrade_provisioning_failure_to_anonymous( + people, token, monkeypatch, mode, outage, status, method +): + from server.transport import mcp + + monkeypatch.setattr(settings, "mcp_auth_mode", mode) + people.unavailable = outage + app = FastAPI() + mcp.attach(app) + client = TestClient(app) + headers = {"Authorization": f"Bearer {token()}"} + if method == "post": + response = client.post( + "/mcp", + headers=headers, + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + ) + else: + response = client.delete("/mcp", headers=headers) + assert response.status_code == status + if outage: + assert response.headers["Retry-After"] == "5" + + +async def test_health_admin_detail_requires_full_person_admission(token): + from server import health + + assert not await health._is_platform_admin(request_for(token())) + + +def test_tenancy_middleware_rechecks_a_wildcard_before_placement(people, token, monkeypatch): + from server.transport.tenancy_gate import TenancyMiddleware + + people.provision("alice@example.test") + people.accounts.append(("acme", "alice@example.test", "missing@example.test", False)) + + async def no_placement(*_, **__): + pytest.fail("an unchecked person must not place an org") + + monkeypatch.setattr(tenancy, "ensure_held", no_placement) + app = FastAPI() + me_routes.attach(app) + app.add_middleware(TenancyMiddleware) + response = TestClient(app).post( + "/v1/me/set-default-namespace", + json={"namespace": "private"}, + headers={"Authorization": f"Bearer {token(org='*')}"}, + ) + assert response.status_code == 401 + + +def test_registered_http_operation_refuses_before_dispatch(people, token, monkeypatch): + from dataclasses import replace + + from server.transport import http + + called = [] + + async def no_dispatch(*_, **__): + called.append(True) + pytest.fail("an unprovisioned token must not reach operation dispatch") + + operation = next(op for op in http.ALL_OPERATIONS if op.name == "memory.get") + + async def no_placement(*_, **__): + pytest.fail("an unchecked HTTP person must not place an org") + + monkeypatch.setattr(http, "ALL_OPERATIONS", [replace(operation, handler=no_dispatch)]) + monkeypatch.setattr(tenancy, "ensure_held", no_placement) + app = FastAPI() + app.include_router(http.build_router()) + payload = {"org": "acme", "namespace": "private", "agent": "agent", "memory_key": "key"} + operation.input_model.model_validate(payload) + response = TestClient(app).post( + "/v1/memory.get", json=payload, headers={"Authorization": f"Bearer {token()}"} + ) + assert response.status_code == 401 + assert not called + + +async def test_actual_signed_jwt_disjoint_graphs_share_one_subject_budget(people, token): + roots = [] + for graph in range(65): + email = f"p{graph}@example.test" + people.provision(email) + names = [f"sa:personal:{graph}-{node}" for node in range(16)] + roots.append(names[0]) + for current, parent in zip(names, [*names[1:], email], strict=True): + people.services[current] = {"created_by": parent, "lineage": []} + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(token(roots[0], parents=roots[1:]), secret=SECRET) + + +async def test_actual_signed_jwt_root_bytes_refuse_before_store_reads(people, token): + with pytest.raises(jwt.TokenError): + await jwt.authenticate_token(token(parents=("x" * 65537,)), secret=SECRET) + assert not people.reads + + +def test_live_raw_verifier_call_census_has_only_the_admission_and_logout_exceptions(): + root = Path(__file__).resolve().parents[2] + hits = set() + for path in (root / "server").rglob("*.py"): + tree = ast.parse(path.read_text()) + imports = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + for item in node.names: + imports[item.asname or item.name] = node.module + "." + item.name + elif isinstance(node, ast.Import): + for item in node.names: + imports[item.asname or item.name.split(".")[0]] = ( + item.name if item.asname else item.name.split(".")[0] + ) + + def name(node, aliases=imports, is_jwt=path.name == "jwt.py"): + if isinstance(node, ast.Name): + return aliases.get(node.id, "server.auth.jwt." + node.id if is_jwt else node.id) + if isinstance(node, ast.Attribute): + return name(node.value) + "." + node.attr + return "" + + for function in ast.walk(tree): + if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for node in ast.walk(function): + if isinstance(node, ast.Call) and name(node.func) == "server.auth.jwt.verify_token": + hits.add((str(path.relative_to(root)), function.name)) + assert hits == { + ("server/auth/jwt.py", "authenticate_token"), + ("server/auth/oauth.py", "revoke_token"), + } diff --git a/tests/unit/test_scim_cell_authorization.py b/tests/unit/test_scim_cell_authorization.py index c87ca87a75..4da6ee6afd 100644 --- a/tests/unit/test_scim_cell_authorization.py +++ b/tests/unit/test_scim_cell_authorization.py @@ -4,6 +4,7 @@ import functools import hmac +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -40,7 +41,9 @@ def identity(monkeypatch): @pytest.fixture def client(identity, store, monkeypatch): monkeypatch.setattr( - scim_groups, "verify_token", functools.partial(verify_token, secret=JWT_SECRET) + scim_groups, + "authenticate_token", + AsyncMock(side_effect=functools.partial(verify_token, secret=JWT_SECRET)), ) api = FastAPI() scim_groups.attach(api) @@ -131,7 +134,7 @@ def test_reserved_cell_token_cannot_fall_through_as_a_groups_admin(client, monke def forbidden(_token): pytest.fail("a reserved SCIM credential must not use the org-admin alternative") - monkeypatch.setattr(scim_groups, "verify_token", forbidden) + monkeypatch.setattr(scim_groups, "authenticate_token", AsyncMock(side_effect=forbidden)) response = client.get("/scim/v2/Groups", headers={"Authorization": f"Bearer {token}"}) assert response.status_code == 401 @@ -142,6 +145,6 @@ def test_groups_availability_failure_cannot_use_an_admin_fallback(client, identi def forbidden(_token): pytest.fail("an unread SCIM cell must not fall through unchecked") - monkeypatch.setattr(scim_groups, "verify_token", forbidden) + monkeypatch.setattr(scim_groups, "authenticate_token", AsyncMock(side_effect=forbidden)) response = client.get("/scim/v2/Groups", headers={"Authorization": "Bearer org-admin"}) assert response.status_code == 503 and response.headers["Retry-After"] == "5" diff --git a/tests/unit/test_scim_erasure_identity_writes.py b/tests/unit/test_scim_erasure_identity_writes.py new file mode 100644 index 0000000000..c907ab8ba4 --- /dev/null +++ b/tests/unit/test_scim_erasure_identity_writes.py @@ -0,0 +1,291 @@ +"""Erasure directory rewrites discard the complete person decision memo.""" + +from __future__ import annotations + +import asyncio +import json +import time +from contextlib import asynccontextmanager +from datetime import UTC, datetime + +import pytest + +from server.auth import delegation, revocation, scim, scim_people, scim_routes +from server.auth.delegation import delegate +from server.auth.jwt import AuthInfo +from server.core import ch +from server.core.principals import USER, personal_name, principal_id +from server.erasure.app import saga_steps +from server.identity.api.models import NamespaceRegisterInput +from server.identity.app import me_service_accounts +from server.identity.app.namespace import registry_row +from server.operations.base import Role +from server.settings import settings +from tests.unit.test_scim_people import People +from tests.unit.test_scim_people import people as people + +ORG = "acme" +USER_ID = "alice@example.test" +OWNER = USER + USER_ID +SA = "sa:erasure-child-123abc" +# Capture at collection, before the function-scoped autouse parent-policy stub. +_REFUSE_A_REVOKED_PARENT = delegation._refuse_a_revoked_parent + + +@asynccontextmanager +async def claim_edge(*args, **kwargs): + yield + + +class Directory(People): + """The I/O edges share FINAL directory rows, not independent policy answers.""" + + def __init__(self, rows, observed): + super().__init__() + self.rows = {row["namespace"]: dict(row) for row in rows} + self.tree = {row["namespace"]: OWNER for row in rows if row["kind"] == "collection"} + self.observed = observed + self.failure = None + self.auth = None + self.writes = [] + self.query_parameters = [] + self.expected_revocation_keys = [] + self.revocation_reads = [] + + def selected(self, table, params): + if table == "loom.collections": + return [ns for ns, owner in self.tree.items() if owner == params["p"]] + if "ns" in params: + return [ + ns + for ns, row in self.rows.items() + if ns == params["ns"] and row["org"] == params["org"] and row["active"] + ] + return [ + ns + for ns, row in self.rows.items() + if row["org"] == params["org"] + and row["kind"] == "collection" + and row["owner_email"] == params["p"] + ] + + def sync_accounts(self): + self.accounts = [ + (row["org"], row["owner_email"], row["contact_email"], not row["password_hash"]) + for row in self.rows.values() + if row["active"] + ] + + async def query_all(self, sql, params=None): + params = params or {} + self.query_parameters.append(params) + if "FROM loom.revoked_tokens" in sql: + assert "WHERE jti IN {keys:Array(String)} AND expires_at > now()" in sql + assert self.expected_revocation_keys + assert params == {"keys": self.expected_revocation_keys} + self.revocation_reads.append(list(params["keys"])) + return [] # the store holds no unexpired revocation for these exact keys + if "UNION DISTINCT" in sql and "loom.collections" in sql: + assert params == {"org": ORG, "p": OWNER} + names = set(self.selected("loom.namespaces", params)) | set(self.tree) + return [{"namespace": ns} for ns in sorted(names)] + self.sync_accounts() + return await super().query_all(sql, params=params) + + async def query_one(self, sql, params=None): + params = params or {} + self.query_parameters.append(params) + if sql.startswith("SELECT count() AS n FROM "): + table = sql.split()[5] + assert table in ("loom.namespaces", "loom.collections") + return {"n": len(self.selected(table, params))} + if "max(version) + 1" in sql: + assert "loom.service_accounts" in sql + return {"v": 1} + if "SELECT sa_subject, created_by, lineage" in sql: + assert params == {"org": ORG, "name": "erasure-child"} + return None + self.sync_accounts() + return await super().query_one(sql, params=params) + + async def insert(self, table, rows, **kwargs): + assert table == "loom.service_accounts" + for row in rows: + self.services[row["sa_subject"]] = dict(row) + + async def execute(self, sql, params=None, **kwargs): + params = params or {} + assert sql.startswith("INSERT INTO ") and "SELECT * REPLACE" in sql + table = sql.split()[2] + assert table in ("loom.namespaces", "loom.collections") + selected = self.selected(table, params) + assert selected + self.writes.append((table, sql, dict(params), tuple(selected))) + if table == "loom.collections": + assert "'' AS owner" in sql + for ns in selected: + self.tree[ns] = "" + return + # Observe the actual cache path during the actual writer's I/O call. + # Nothing clears or edits the memo between any of these checks. + reads = len(self.reads) + self.observed["during_refused"] = await scim.refuses_auth(self.auth) + self.observed["during_read_delta"] = len(self.reads) - reads + assert "'' AS owner_email" in sql + for ns in selected: + row = self.rows[ns] + row["owner_email"] = "" + if "false AS active" in sql: + row["active"] = False + row["description"] = "" + row["title"] = "" + row["updated_at"] = row["updated_at"].replace(year=2099) + self.observed["rewritten"] = len(selected) + if self.failure is not None: + raise self.failure + + +def directory_rows(kind, count): + now = datetime(2026, 1, 1, tzinfo=UTC) + rows = [] + for i in range(count): + ns = personal_name(ORG, OWNER) if kind == "personal" and i == 0 else f"col-audit-{i}" + # The same builder and inputs used by collection.create and first-write + # personal auto-registration: contacts and passwords are empty. + row = registry_row( + NamespaceRegisterInput(org=ORG, namespace=ns, owner_email=OWNER), None, now + ) + assert row["kind"] == ("personal" if ns.startswith("col-me-") else "collection") + assert row["contact_email"] == row["password_hash"] == "" + rows.append(row) + return rows + + +async def setup_delegated_world(people, monkeypatch, kind, count): + assert principal_id(OWNER, USER) == USER_ID + monkeypatch.setattr(settings, "grant_cache_ttl_seconds", 240) + observed = {} + world = Directory(directory_rows(kind, count), observed) + _, email, _ = scim_routes.user_name_and_email( + {"userName": OWNER, "emails": [{"value": OWNER, "type": "work", "primary": True}]} + ) + assert email == OWNER + world.provision(email) + monkeypatch.setattr(ch, "get_client", lambda: world) + monkeypatch.setattr(saga_steps, "account_change", claim_edge) + monkeypatch.setattr(saga_steps, "collections_change", claim_edge) + # This is the AuthInfo shape returned by the verified end-user issuer path. + # Its normal delegate/save path creates a nonexempt native SA with the user + # in creator, parent_subjects and persisted lineage. + monkeypatch.setattr(delegation, "_refuse_a_revoked_parent", _REFUSE_A_REVOKED_PARENT) + assert delegation._refuse_a_revoked_parent is _REFUSE_A_REVOKED_PARENT + assert _REFUSE_A_REVOKED_PARENT.__module__ == "server.auth.delegation" + assert _REFUSE_A_REVOKED_PARENT.__name__ == "_refuse_a_revoked_parent" + parent = AuthInfo( + subject=OWNER, + org=ORG, + namespaces=(), + roles=(Role.VIEWER,), + is_service_account=False, + impersonated_by="https://issuer.example.test", + read_scope="own_user", + read_user_id=USER_ID, + token_id="unit-end-user-token", + issued_at=int(time.time()), + ) + world.expected_revocation_keys = sorted( + key + for subject in (OWNER, parent.token_id) + for key in ( + subject, + revocation.CUTOFF_PREFIX + subject, + revocation.SCIM_PREFIX + subject, + ) + ) + child = await delegate( + parent, + org=ORG, + requested={"*": None}, + scoped=False, + ttl_seconds=60, + is_service_account=True, + ) + assert world.revocation_reads == [world.expected_revocation_keys] + await me_service_accounts.save_token( + "erasure-child", SA, child, created_by=parent.subject, now=datetime.now(UTC) + ) + world.auth = child.as_auth(SA) + assert world.auth.subject == SA and world.auth.creator == OWNER + assert OWNER in world.auth.parent_subjects and OWNER in world.services[SA]["lineage"] + assert world.services[SA]["revoked_at"] is None + assert world.auth.impersonated_by == "" + assert len(world.rows) == count + assert sum(len(row["contact_email"].encode()) for row in world.rows.values()) == 0 + return world, observed + + +async def cold_decision(world): + auth = world.auth + roots = (auth.subject, *sorted(set((*auth.parent_subjects, auth.creator)))) + decision = scim_people._Decision(ORG, auth.subject, "", scim_people._now() + 30) + try: + result = await decision.refuses(roots) + except scim_people._Refused: + result = True # the public whole-decision wrapper maps supported-size overflow to refusal + assert decision.transferred < 65_536 + assert all( + len(json.dumps(params, ensure_ascii=False, separators=(",", ":")).encode()) < 65_536 + for params in world.query_parameters + ) + return result + + +async def rewrite(kind): + if kind == "personal": + return await saga_steps._step_retire_personal_collection_by_user(ORG, USER_ID) + return await saga_steps._step_forget_collection_owner_by_user(ORG, USER_ID) + + +@pytest.mark.parametrize("kind", ["personal", "collection"]) +@pytest.mark.parametrize("outcome", ["success", "lost-reply", "cancel"]) +async def test_erasure_rechecks_native_ancestry_after_account_budget_shrinks( + people, monkeypatch, kind, outcome +): + world, observed = await setup_delegated_world(people, monkeypatch, kind, 1025) + assert await scim.refuses_auth(world.auth) is True + assert await cold_decision(world) is True + # Capture the same exception object through the app and store layers. + if outcome != "success": + world.failure = ( + ConnectionError("directory rewrite reply lost") + if outcome == "lost-reply" + else asyncio.CancelledError() + ) + with pytest.raises(type(world.failure)) as caught: + await rewrite(kind) + assert caught.value is world.failure + else: + await rewrite(kind) + assert observed["rewritten"] == (1 if kind == "personal" else 1025) + assert await cold_decision(world) is False, ( + "the actual rewrite must leave a qualifying bounded graph" + ) + assert await scim.refuses_auth(world.auth) is False, ( + "the old account-overflow refusal must not survive the local rewrite" + ) + assert observed["during_refused"] is True + assert observed["during_read_delta"] > 0, ( + "pre-attempt invalidation must prevent a warm answer during the write" + ) + assert len(world.writes) == (1 if kind == "personal" else 2) + + +@pytest.mark.parametrize("kind", ["personal", "collection"]) +async def test_erasure_preserves_a_qualifying_native_ancestry_control(people, monkeypatch, kind): + world, observed = await setup_delegated_world(people, monkeypatch, kind, 1024) + assert await cold_decision(world) is False + assert await scim.refuses_auth(world.auth) is False + await rewrite(kind) + assert await cold_decision(world) is False + assert await scim.refuses_auth(world.auth) is False + assert observed["rewritten"] == (1 if kind == "personal" else 1024) diff --git a/tests/unit/test_scim_fresh_reader_bounds.py b/tests/unit/test_scim_fresh_reader_bounds.py new file mode 100644 index 0000000000..55f9cecef5 --- /dev/null +++ b/tests/unit/test_scim_fresh_reader_bounds.py @@ -0,0 +1,55 @@ +"""Fresh ancestry reads share the complete decision's parameter budget.""" + +from __future__ import annotations + +import json + +import pytest + +from server.auth import scim, service_accounts +from server.auth.scim_people import MAX_BYTES +from tests.unit.test_scim_people import People +from tests.unit.test_scim_people import people as people + + +def _parameter_bytes(params: dict) -> int: + return len(json.dumps(params, ensure_ascii=False, separators=(",", ":")).encode()) + + +@pytest.mark.parametrize("over", [0, 1, 13], ids=["limit", "one-over", "original-counterexample"]) +async def test_complete_graph_bounds_the_actual_fresh_reader_parameters( + people: People, over: int +) -> None: + # Accepted roots put the reader's fixed parameters exactly at the limit. + subject = "sa:" + "s" * 32750 + org = "o" * (32737 + over) + people.services[subject] = {"created_by": "alice@example.test", "lineage": []} + people.provision("alice@example.test") + + refused = await scim.refuses_subject(subject, org=org) + + sizes = [_parameter_bytes(params) for params in people.reads] + assert max(sizes, default=0) <= MAX_BYTES + reader = [ + params for params in people.reads if params.get("sub") == subject and "items" in params + ] + assert refused is (over > 0) + if over: + assert reader == [] + else: + assert len(reader) == 1 + assert _parameter_bytes(reader[0]) == MAX_BYTES + + +async def test_existing_inheritance_reader_keeps_its_query_contract( + people: People, monkeypatch: pytest.MonkeyPatch +) -> None: + subject, org = "sa:" + "s" * 32750, "o" * 32750 + people.services[subject] = {"created_by": "alice@example.test", "lineage": []} + monkeypatch.setattr(service_accounts, "_creators", {}) + + creator = await service_accounts.sa_creator(org, subject) + + assert creator is not None and creator.subject == "alice@example.test" + assert len(people.reads) == 1 + assert _parameter_bytes(people.reads[0]) == MAX_BYTES + 13 diff --git a/tests/unit/test_scim_groups.py b/tests/unit/test_scim_groups.py index ac1a11f460..94fa476107 100644 --- a/tests/unit/test_scim_groups.py +++ b/tests/unit/test_scim_groups.py @@ -6,6 +6,7 @@ import functools from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -65,7 +66,11 @@ async def _emit(*_a: Any, **_k: Any) -> str: return "h" monkeypatch.setattr(scim_groups.audit, "emit", _emit) - monkeypatch.setattr(scim_groups, "verify_token", functools.partial(verify_token, secret=SECRET)) + monkeypatch.setattr( + scim_groups, + "authenticate_token", + AsyncMock(side_effect=functools.partial(verify_token, secret=SECRET)), + ) return s diff --git a/tests/unit/test_scim_identity_writes.py b/tests/unit/test_scim_identity_writes.py new file mode 100644 index 0000000000..506e06323b --- /dev/null +++ b/tests/unit/test_scim_identity_writes.py @@ -0,0 +1,173 @@ +"""Identity writes invalidate complete decisions before and after partial writes.""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime + +import pytest + +from server.auth import machines, scim, self_signup, sso_links +from server.auth.delegation import Delegated +from server.core import grant_cache, versions +from server.identity.app import me_service_accounts +from server.settings import settings +from tests.unit.test_scim_people import people as people + +EMAIL = "alice@example.test" + + +@asynccontextmanager +async def announced(*args, **kwargs): + yield + + +@pytest.mark.parametrize("outcome", ["success", "lost-reply", "cancel"]) +@pytest.mark.parametrize("kind", ["account", "service-account", "sso-link", "machine", "scim"]) +async def test_real_identity_writers_discard_decisions_before_and_after_the_write( + people, monkeypatch, kind, outcome +): + monkeypatch.setattr(settings, "grant_cache_ttl_seconds", 240) + people.provision(EMAIL) + people.accounts.append(("acme", "alice", EMAIL, kind != "sso-link")) + + async def next_version(*args, **kwargs): + return 2 + + async def no_side_effect(*args, **kwargs): + return None + + monkeypatch.setattr(versions, "next_version", next_version) + monkeypatch.setattr(machines, "next_version", next_version) + monkeypatch.setattr(scim, "next_version", next_version) + monkeypatch.setattr(sso_links, "next_version", next_version) + monkeypatch.setattr(scim, "_audit_transition", no_side_effect) + monkeypatch.setattr(self_signup, "insert_owner_grant", no_side_effect) + monkeypatch.setattr(me_service_accounts, "hand_down_lineage", no_side_effect) + monkeypatch.setattr(grant_cache, "announced", announced) + subject = { + "account": "alice", + "service-account": "sa:personal:one", + "sso-link": "alice", + "machine": "sa:robot:agent", + "scim": EMAIL, + }[kind] + before = kind in ("service-account", "sso-link") + if kind == "machine": + people.bots["robot"] = True + assert await scim.refuses_subject(subject, org="acme") is before + warmed_reads = len(people.reads) + expected_error = ( + ConnectionError("identity row landed but the reply was lost") + if outcome == "lost-reply" + else asyncio.CancelledError() + ) + writes = [] + + async def insert(table, rows, **kwargs): + assert await scim.refuses_subject(subject, org="acme") is before + assert len(people.reads) > warmed_reads, "a write must start with a fresh complete decision" + writes.extend((table, dict(row)) for row in rows) + for row in rows: + if table == "loom.namespaces": + people.accounts.append( + (row["org"], row["owner_email"], row["contact_email"], not row["password_hash"]) + ) + elif table == "loom.service_accounts": + people.services[row["sa_subject"]] = dict(row) + elif table == "loom.sso_links": + people.accounts[:] = [ + (org, owner, email, bool(row["linked"])) + for org, owner, email, _ in people.accounts + ] + elif table == "loom.machine_identities": + people.bots[row["name"]] = bool(row["active"]) + elif table == "loom.scim_users": + people.users[row["id"]] = dict(row) + else: + pytest.fail(f"unexpected identity write: {table}") + if outcome != "success": + raise expected_error + + monkeypatch.setattr(people, "insert", insert, raising=False) + + async def write(): + if kind == "account": + await self_signup.provision_account( + org="acme", + namespace="new-account", + subject="alice", + contact_email="missing@example.test", + password_hash="unit-password-hash", + ch=people, + ) + elif kind == "service-account": + delegated = Delegated( + org="acme", + namespaces=["private"], + roles=["writer"], + ns_roles=None, + write_namespaces=None, + read_scope="", + read_user_id="", + ttl_seconds=60, + parents=[], + parent_subjects=[], + parent_iats=[], + creator="alice", + is_service_account=True, + ) + await me_service_accounts.save_token( + "personal", subject, delegated, created_by="alice", now=datetime.now(UTC) + ) + elif kind == "sso-link": + await sso_links._write(EMAIL, [("acme", "private")], "alice", linked=1, ch=people) + elif kind == "machine": + await machines.save( + { + "name": "robot", + "org": "acme", + "ns_roles": [], + "public_keys": "", + "issuer": "", + "audience": "", + "claims": {}, + "agent_claim": "", + "jwks_uri": "", + "max_token_s": 60, + "max_agents": 1, + "max_access_token_s": 60, + "platform_admin": False, + }, + by="alice", + active=False, + ch=people, + ) + else: + row = { + "id": scim.user_id(EMAIL), + "email": EMAIL, + "user_name": EMAIL, + "emails": json.dumps([{"value": EMAIL, "type": "work", "primary": True}]), + "addresses": [EMAIL], + "work_addresses": [EMAIL], + "active": True, + "deleted": False, + "subjects": [EMAIL], + "created_at": datetime.now(UTC), + "updated_at": datetime.now(UTC), + "version": 1, + } + await scim._write_audited(row, {**row, "active": False}) + + if outcome == "success": + await write() + else: + with pytest.raises(type(expected_error)) as caught: + await write() + assert caught.value is expected_error + assert len(writes) == 1 + assert await scim.refuses_subject(subject, org="acme") is not before + assert len(people.reads) > warmed_reads diff --git a/tests/unit/test_scim_input_bounds.py b/tests/unit/test_scim_input_bounds.py new file mode 100644 index 0000000000..4548e45976 --- /dev/null +++ b/tests/unit/test_scim_input_bounds.py @@ -0,0 +1,57 @@ +"""Every decision input is bounded before memo keys or storage reads.""" + +from __future__ import annotations + +import pytest + +from server.auth import authz, scim +from server.settings import settings +from tests.unit.test_scim_people import people as people + + +@pytest.mark.parametrize("required", [False, True]) +@pytest.mark.parametrize( + "org", + [None, [], {}, 1, "x" * 65537, "é" * 32769, "\x00", "\ud800"], + ids=["none", "list", "dict", "number", "characters", "utf8", "nul", "surrogate"], +) +async def test_invalid_org_refuses_before_key_construction_or_store_reads( + people, monkeypatch, org, required +): + monkeypatch.setattr(settings, "cell", "first" if required else "") + + async def no_memo(*_, **__): + pytest.fail("invalid decision inputs must refuse before constructing a memo key") + + monkeypatch.setattr(authz, "_remembered", no_memo) + assert await scim.refuses_subject("owner", org=org) is True + assert not people.reads + + +@pytest.mark.parametrize("required", [False, True]) +@pytest.mark.parametrize( + "subject,org", + [("x" * 26000, "o" * 40000), ("owner", '"' * 32765)], + ids=["aggregate-utf8", "aggregate-serialized"], +) +async def test_org_shares_the_same_utf8_and_serialized_input_budget( + people, monkeypatch, subject, org, required +): + monkeypatch.setattr(settings, "cell", "first" if required else "") + + async def no_memo(*_, **__): + pytest.fail("the aggregate input budget must be checked before the memo") + + monkeypatch.setattr(authz, "_remembered", no_memo) + assert await scim.refuses_subject(subject, org=org) is True + assert not people.reads + + +@pytest.mark.parametrize("required", [False, True]) +@pytest.mark.parametrize("org", ["", "*", "acme"]) +async def test_valid_org_values_keep_the_existing_required_and_standalone_contract( + people, monkeypatch, org, required +): + monkeypatch.setattr(settings, "cell", "first" if required else "") + people.provision("alice@example.test") + assert await scim.refuses_subject("alice@example.test", org=org) is False diff --git a/tests/unit/test_scim_people.py b/tests/unit/test_scim_people.py new file mode 100644 index 0000000000..41afdf7357 --- /dev/null +++ b/tests/unit/test_scim_people.py @@ -0,0 +1,444 @@ +"""Provisioning follows each person's verified account and the complete ancestry.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + +import pytest + +from server.auth import authz, login, machines, scim +from server.core import cell, ch +from server.settings import settings + +_DEACTIVATED = scim.deactivated + + +@dataclass +class People: + accounts: list[tuple[str, str, str, bool]] = field(default_factory=list) + users: dict[str, dict[str, Any]] = field(default_factory=dict) + services: dict[str, dict[str, Any]] = field(default_factory=dict) + bots: dict[str, bool] = field(default_factory=dict) + reads: list[dict[str, Any]] = field(default_factory=list) + unavailable: bool = False + on_read: Callable[[], None] | None = None + + def provision(self, email: str, *, active=True, deleted=False, work=()): + email = scim.normalized(email) + self.users[scim.user_id(email)] = { + "id": scim.user_id(email), + "email": email, + "active": active, + "deleted": deleted, + "work_addresses": set(work), + } + + async def query_all(self, sql, params=None): + p = params or {} + self.reads.append(p) + if self.on_read is not None: + self.on_read() + if self.unavailable: + raise ConnectionError("identity store unavailable") + if "loom.namespaces" in sql and "SELECT DISTINCT" in sql: + owners = { + owner.lower() + for org, owner, email, verified in self.accounts + if org == p["org"] + and (owner.lower() == p["sub"] or (email.lower() == p["sub"] and verified)) + } + return [ + { + "owner": o if len(o.encode()) <= p["bytes"] else "", + "oversized": len(o.encode()) > p["bytes"], + } + for o in sorted(owners)[:2] + ] + if "loom.scim_users" in sql and "missing" not in p: + return [self.users[uid] for uid in p["ids"] if uid in self.users] + raise AssertionError(sql) + + async def query_one(self, sql, params=None): + p = params or {} + self.reads.append(p) + if self.on_read is not None: + self.on_read() + if self.unavailable: + raise ConnectionError("identity store unavailable") + if "loom.namespaces" in sql: + accounts = [ + (email.lower(), verified) + for org, owner, email, verified in self.accounts + if org == p["org"] and owner.lower() == p["owner"] + ] + size = sum(len(email.encode()) for email, _ in accounts) + return { + "account_count": len(accounts), + "account_bytes": size, + "accounts": accounts if len(accounts) <= p["items"] and size <= p["bytes"] else [], + } + if "loom.machine_identities" in sql: + return {"active": self.bots[p["name"]]} if p["name"] in self.bots else None + if "loom.service_accounts" in sql: + row = self.services.get(p["sub"]) + if row is None: + return None + above = row.get("lineage", []) + creator = row.get("created_by", "") + size = sum(len(s.encode()) for s in above) + len(creator.encode()) + return { + "lineage_count": len(above), + "subject_bytes": size, + "inherit_creator_grants": False, + "kind": "personal", + "revoked_at": row.get("revoked_at"), + "expires_at": row.get("expires_at"), + "bounded_creator": creator if len(creator.encode()) <= p["bytes"] else "", + "bounded_lineage": above if len(above) <= p["items"] and size <= p["bytes"] else [], + } + if "loom.scim_users" in sql and "missing" in p: + qualified = set() + for uid in p["ids"]: + row = self.users.get(uid) + if row and row["active"] and not row["deleted"] and row["email"] in p["witnesses"]: + qualified.update(row["work_addresses"] & set(p["missing"])) + return {"qualified": sorted(qualified)} + if "loom.scim_users" in sql: + row = self.users.get(p["id"]) + return {"x": 1} if row and not row["active"] else None + raise AssertionError(sql) + + +@pytest.fixture +def people(monkeypatch): + world = People() + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", "lsc_first_" + "a" * 64) + monkeypatch.setattr(cell, "mode", lambda: "enforced") + monkeypatch.setattr(cell, "deployment_id", lambda: "unit-deployment") + monkeypatch.setattr(ch, "get_client", lambda: world) + monkeypatch.setattr(scim, "get_client", lambda: world) + monkeypatch.setattr(scim, "deactivated", _DEACTIVATED) + monkeypatch.setattr(machines, "get_client", lambda: world) + monkeypatch.setattr(authz, "_ENDED", {}) + return world + + +@pytest.mark.parametrize( + "status,refused", [("active", False), ("missing", True), ("inactive", True), ("deleted", True)] +) +async def test_every_person_needs_a_live_own_id(people, status, refused): + if status != "missing": + people.provision( + "alice@example.test", active=status != "inactive", deleted=status == "deleted" + ) + assert await scim.refuses_subject("alice@example.test", org="acme") is refused + + +@pytest.mark.parametrize("verified,refused", [(False, True), (True, False)]) +async def test_password_contact_cannot_supply_ownership(people, verified, refused): + people.accounts.append(("acme", "outsider", "assigned@example.test", verified)) + people.provision("assigned@example.test") + assert await scim.refuses_subject("outsider", org="acme") is refused + + +@pytest.mark.parametrize("subject", ["alice", "first@example.test"]) +async def test_verified_spelling_checks_every_owned_account(people, subject): + people.accounts.extend( + [ + ("acme", "alice", "first@example.test", True), + ("acme", "alice", "second@example.test", False), + ] + ) + people.provision("first@example.test") + assert await scim.refuses_subject(subject, org="acme") + + +@pytest.mark.parametrize("subject", ["owner@example.test", "contact@example.test"]) +@pytest.mark.parametrize("owner_state", ["missing", "inactive", "deleted", "work"]) +async def test_email_canonical_owner_is_mandatory_and_an_alias_witness( + people, subject, owner_state +): + owner, contact = "owner@example.test", "contact@example.test" + people.accounts.append(("acme", owner, contact, True)) + if owner_state == "work": + people.provision(owner, work=(contact,)) + else: + people.provision(contact) + if owner_state != "missing": + people.provision( + owner, active=owner_state != "inactive", deleted=owner_state == "deleted" + ) + assert await scim.refuses_subject(subject, org="acme") is (owner_state != "work") + + +async def test_tombstoned_own_id_cannot_borrow_a_work_alias(people): + people.accounts.append(("acme", "alice", "primary@example.test", True)) + people.accounts.append(("acme", "alice", "alias@example.test", True)) + people.provision("primary@example.test", work=("alias@example.test",)) + people.provision("alias@example.test", deleted=True) + assert await scim.refuses_subject("alice", org="acme") + + +async def test_unrelated_primary_cannot_lend_its_work_alias(people): + people.provision("unrelated@example.test", work=("alice@example.test",)) + assert await scim.refuses_subject("alice@example.test", org="acme") + + +async def test_ambiguous_owners_refuse(people): + people.accounts.extend( + [ + ("acme", "alice", "shared@example.test", True), + ("acme", "bob", "shared@example.test", True), + ] + ) + people.provision("shared@example.test") + assert await scim.refuses_subject("shared@example.test", org="acme") + + +@pytest.mark.parametrize("org", ["other", "*", ""]) +async def test_account_ownership_never_crosses_orgs(people, org): + people.accounts.append(("acme", "alice", "alice@example.test", True)) + people.provision("alice@example.test") + assert await scim.refuses_subject("alice", org=org) + + +@pytest.mark.parametrize( + "also", [("missing@example.test",), ("alice@example.test", "missing@example.test")] +) +async def test_every_additional_root_qualifies_independently(people, also): + people.provision("alice@example.test") + assert await scim.refuses_subject("alice@example.test", org="acme", also=also) + + +async def test_primary_verified_email_does_not_provision_another_root(people): + people.provision("alice@example.test") + assert await scim.refuses_subject( + "alice", org="acme", verified_email="alice@example.test", also=("outsider",) + ) + + +async def test_service_account_cannot_lend_a_verified_email_to_its_creator(people): + people.provision("assigned@example.test") + people.services["sa:personal:one"] = {"created_by": "outsider", "lineage": []} + assert await scim.refuses_subject( + "sa:personal:one", org="acme", verified_email="assigned@example.test" + ) + + +@pytest.mark.parametrize("inherits", [False, True]) +async def test_every_personal_service_account_ancestor_qualifies(people, inherits): + people.provision("alice@example.test") + people.services["sa:personal:one"] = { + "created_by": "alice@example.test", + "lineage": ["missing@example.test"], + "inherit_creator_grants": inherits, + } + assert await scim.refuses_subject("sa:personal:one", org="acme") + + +async def test_shared_ancestor_is_not_a_cycle(people): + people.provision("alice@example.test") + people.services["sa:personal:one"] = {"created_by": "alice@example.test", "lineage": []} + people.services["sa:personal:two"] = {"created_by": "alice@example.test", "lineage": []} + assert not await scim.refuses_subject("sa:personal:one", org="acme", also=("sa:personal:two",)) + + +async def test_a_cycle_in_a_later_root_refuses(people): + people.provision("alice@example.test") + people.services["sa:personal:one"] = {"created_by": "sa:personal:two", "lineage": []} + people.services["sa:personal:two"] = {"created_by": "sa:personal:one", "lineage": []} + assert await scim.refuses_subject("alice@example.test", org="acme", also=("sa:personal:one",)) + + +@pytest.mark.parametrize("extra", [1023, 1024]) +async def test_repeated_roots_count_before_deduplication(people, extra): + people.provision("alice@example.test") + assert await scim.refuses_subject( + "alice@example.test", org="acme", also=("alice@example.test",) * extra + ) is (extra == 1024) + if extra == 1024: + assert not people.reads + + +async def test_root_bytes_refuse_before_reads(people): + assert await scim.refuses_subject("x" * 65537, org="acme") + assert not people.reads + + +async def test_a_larger_graph_cannot_reuse_a_single_root_permit(people): + people.provision("alice@example.test") + people.provision("bob@example.test") + assert not await scim.refuses_subject("alice@example.test", org="acme") + people.provision("alice@example.test", active=False) + assert await scim.refuses_subject("alice@example.test", org="acme", also=("bob@example.test",)) + + +@pytest.mark.parametrize("stored_first", [False, True]) +async def test_memo_includes_the_verified_session_email(people, stored_first): + people.provision("alice@example.test") + emails = ["missing@example.test", ""] if stored_first else ["", "missing@example.test"] + answers = [ + await scim.refuses_subject("alice@example.test", org="acme", verified_email=email) + for email in emails + ] + assert answers == [bool(email) for email in emails] + + +async def test_extra_root_order_has_one_complete_memo_identity(people): + for email in ("alice@example.test", "bob@example.test", "carol@example.test"): + people.provision(email) + assert not await scim.refuses_subject( + "alice@example.test", org="acme", also=("bob@example.test", "carol@example.test") + ) + people.unavailable = True + assert not await scim.refuses_subject( + "alice@example.test", org="acme", also=("carol@example.test", "bob@example.test") + ) + + +async def test_outage_is_retryable_and_never_cached(people): + people.unavailable = True + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject("alice@example.test", org="acme") + people.unavailable = False + people.provision("alice@example.test") + assert not await scim.refuses_subject("alice@example.test", org="acme") + + +@pytest.mark.parametrize("scim_on", [False, True]) +async def test_standalone_keeps_existing_deactivation_behavior(people, monkeypatch, scim_on): + monkeypatch.setattr(settings, "cell", "") + if not scim_on: + monkeypatch.setattr(settings, "scim_token", "") + people.provision("disabled@example.test", active=False) + assert not await scim.refuses_subject("missing@example.test", org="acme") + assert await scim.refuses_subject("disabled@example.test", org="acme") + + +async def test_account_row_overflow_never_checks_a_prefix(people): + people.accounts.extend(("acme", "alice", "alice@example.test", True) for _ in range(1025)) + people.provision("alice@example.test") + assert await scim.refuses_subject("alice", org="acme") + + +async def test_one_oversized_stored_contact_refuses(people): + people.accounts.append(("acme", "alice", "x" * 65537 + "@example.test", True)) + assert await scim.refuses_subject("alice", org="acme") + + +async def test_disjoint_graphs_share_the_subject_budget(people): + roots = [] + for graph in range(65): + email = f"p{graph}@example.test" + people.provision(email) + names = [f"sa:personal:{graph}-{node}" for node in range(16)] + roots.append(names[0]) + for current, parent in zip(names, [*names[1:], email], strict=True): + people.services[current] = {"created_by": parent, "lineage": []} + assert await scim.refuses_subject(roots[0], org="acme", also=tuple(roots[1:])) + + +async def test_repeated_edges_count_as_work_even_when_the_person_is_shared(people): + people.provision("alice@example.test") + people.services["sa:personal:one"] = { + "created_by": "alice@example.test", + "lineage": ["alice@example.test"] * 1024, + } + assert await scim.refuses_subject("sa:personal:one", org="acme") + + +async def test_security_does_not_reuse_grant_inheritance_metadata(people, monkeypatch): + import time + + from server.auth import service_accounts + + people.provision("alice@example.test") + people.services["sa:personal:one"] = {"created_by": "missing@example.test", "lineage": []} + monkeypatch.setattr( + service_accounts, + "_creators", + { + ("acme", "sa:personal:one"): ( + time.monotonic(), + service_accounts.SACreator("alice@example.test", True), + ) + }, + ) + assert await scim.refuses_subject("sa:personal:one", org="acme") + + +@pytest.mark.parametrize("delta,refused", [(-1, True), (0, True), (1, False)]) +async def test_fresh_service_account_expiry_uses_the_clock(people, monkeypatch, delta, refused): + from types import SimpleNamespace + + from server.auth import scim_people + + people.provision("alice@example.test") + people.services["sa:personal:one"] = { + "created_by": "alice@example.test", + "lineage": [], + "expires_at": datetime.fromtimestamp(1000 + delta, UTC), + } + monkeypatch.setattr( + scim_people, "time", SimpleNamespace(monotonic=lambda: 0.0, time=lambda: 1000.0) + ) + assert await scim.refuses_subject("sa:personal:one", org="acme") is refused + + +@pytest.mark.parametrize("configured", [0, -1, float("inf"), float("nan"), 60]) +async def test_one_finite_deadline_covers_every_root(people, monkeypatch, configured): + from server.auth import scim_people + from tests.unit.clock_fake import ManualClock + + clock = ManualClock() + monkeypatch.setattr(settings, "ch_timeout_read_s", configured) + monkeypatch.setattr(scim_people, "_now", clock.monotonic) + people.on_read = lambda: clock.advance(10) + people.provision("alice@example.test") + people.provision("bob@example.test") + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject("alice@example.test", org="acme", also=("bob@example.test",)) + assert len(people.reads) == 3 + assert clock.t == 30 + + +async def test_a_smaller_positive_read_budget_wins(people, monkeypatch): + from server.auth import scim_people + from tests.unit.clock_fake import ManualClock + + clock = ManualClock() + monkeypatch.setattr(settings, "ch_timeout_read_s", 0.25) + monkeypatch.setattr(scim_people, "_now", clock.monotonic) + people.on_read = lambda: clock.advance(0.25) + people.provision("alice@example.test") + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject("alice@example.test", org="acme") + assert len(people.reads) == 1 + + +async def test_identity_invalidation_clears_a_complete_graph_answer(people): + people.provision("alice@example.test") + assert not await scim.refuses_subject("alice@example.test", org="acme") + people.provision("alice@example.test", active=False) + authz.forget_ended("identity", "alice") + assert await scim.refuses_subject("alice@example.test", org="acme") + + +async def test_local_invalidation_prevents_overtaken_memo_population(people): + people.provision("alice@example.test") + people.on_read = lambda: authz.forget_ended("identity", "alice") + assert not await scim.refuses_subject("alice@example.test", org="acme") + people.on_read = None + people.provision("alice@example.test", active=False) + assert await scim.refuses_subject("alice@example.test", org="acme") + + +async def test_standalone_read_failure_retains_the_sign_in_exception(people, monkeypatch): + monkeypatch.setattr(settings, "cell", "") + people.unavailable = True + with pytest.raises(login.SignInUnavailable): + await scim.refuses_subject("alice@example.test", org="acme") diff --git a/tests/unit/test_scim_sessions.py b/tests/unit/test_scim_sessions.py new file mode 100644 index 0000000000..80748cf0e5 --- /dev/null +++ b/tests/unit/test_scim_sessions.py @@ -0,0 +1,412 @@ +"""Session policy failures preserve credentials and trusted email evidence.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import time + +import pytest +from fastapi.testclient import TestClient + +from server.auth import authz, device, jit, login, oauth, oauth_store, oidc, revocation, scim +from tests.unit.oauth_store_fake import MemoryOAuthStore, oauth_app +from tests.unit.test_scim_people import people as people + +EMAIL = "alice@example.test" +IDENTITY = ("alice", "acme", ["private"], ["writer"]) + + +@pytest.fixture +def session(people, monkeypatch): + store = MemoryOAuthStore() + monkeypatch.setattr(oauth, "store", store) + monkeypatch.setattr(oauth.settings, "oauth_login_provider", "oidc") + people.accounts.append(("acme", "alice", EMAIL, True)) + people.provision(EMAIL) + minted, rotated, revoked, events = [], [], [], [] + + def mint(*args, **kwargs): + minted.append((args, kwargs)) + return "unit-session-access" + + async def rotate(grant, refresh): + rotated.append(grant["upstream_refresh"]) + return "unit-upstream-next" + + async def revoke(value): + revoked.append(value) + + async def not_revoked(*args, **kwargs): + return False + + async def roles(subject, org, original): + return original + + async def emit(event, **kwargs): + events.append((event, kwargs)) + return "" + + monkeypatch.setattr(oauth, "mint_token", mint) + monkeypatch.setattr(oauth, "_ask_identity_provider", rotate) + monkeypatch.setattr(oidc, "revoke", revoke) + monkeypatch.setattr(revocation, "subjects_revoked", not_revoked) + monkeypatch.setattr(revocation, "_revoked_jtis", frozenset()) + monkeypatch.setattr(revocation, "_revoked_before", {}) + monkeypatch.setattr(login, "refreshed_roles", roles) + monkeypatch.setattr(oauth.audit, "emit", emit) + return { + "store": store, + "minted": minted, + "rotated": rotated, + "revoked": revoked, + "events": events, + } + + +def grant(**changes): + return { + "client_id": "unit-client", + "redirect_uri": "https://example.test/callback", + "code_challenge": "challenge", + "subject": "alice", + "org": "acme", + "namespaces": ["private"], + "roles": ["writer"], + "exp": int(time.time()) + oauth._REFRESH_TTL, + "since": int(time.time()), + "family": "unit-family", + "upstream_refresh": "unit-upstream-old", + "verified_email": EMAIL, + **changes, + } + + +async def test_callback_requires_provisioning_before_jit_lookup_or_write( + people, session, monkeypatch +): + async def no_jit(*args, **kwargs): + pytest.fail("an unprovisioned callback must not find or make an account") + + monkeypatch.setattr(jit, "finish_unfinished_account", no_jit) + monkeypatch.setattr(login, "resolve_identity_for_email", no_jit) + result = await oauth._account_for( + "missing@example.test", {"kind": "dashboard"}, "unit-upstream" + ) + assert isinstance(result, oauth._Denied) + assert not session["minted"] + + +async def test_signed_in_identity_carries_trusted_email_into_consent(session): + await session["store"].save_client( + "unit-client", ["https://example.test/callback"], "Unit client" + ) + pending = { + "kind": "connect", + "client_id": "unit-client", + "redirect_uri": "https://example.test/callback", + "code_challenge": "challenge", + "mcp_state": "unit-state", + } + await oauth._signed_in_as(pending, IDENTITY, "unit-upstream", EMAIL) + records = list(session["store"].records("pending").values()) + assert len(records) == 1 + assert records[0]["data"]["verified_email"] == EMAIL + + +async def test_code_records_keep_the_verified_email(session): + await oauth._redirect_with_code( + { + "client_id": "unit-client", + "redirect_uri": "https://example.test/callback", + "code_challenge": "challenge", + }, + IDENTITY, + upstream_refresh="unit-upstream", + state="unit-state", + since=int(time.time()), + verified_email=EMAIL, + ) + records = list(session["store"].records("code").values()) + assert len(records) == 1 + assert records[0]["verified_email"] == EMAIL + + +async def test_issue_tokens_carries_email_into_the_next_refresh(session): + await oauth.issue_tokens(grant(), "unit-client", since=int(time.time())) + saved = list(session["store"].refresh.values()) + assert len(saved) == 1 + assert saved[0]["verified_email"] == EMAIL + + +async def test_stored_email_is_not_omitted_from_refresh_policy(people, session): + original = grant(verified_email="missing@example.test") + await session["store"].save_refresh("unit-refresh", original) + with pytest.raises(oauth.OAuthError) as refused: + await oauth._refresh_token_grant( + {"client_id": "unit-client", "refresh_token": "unit-refresh"} + ) + assert refused.value.status_code != 503 + assert not session["rotated"] and not session["minted"] + assert not session["store"].owed + + +async def test_local_outage_puts_back_identical_refresh_before_upstream_rotation(people, session): + await session["store"].save_refresh("unit-refresh", grant()) + people.unavailable = True + with pytest.raises(oauth.OAuthError) as unavailable: + await oauth._refresh_token_grant( + {"client_id": "unit-client", "refresh_token": "unit-refresh"} + ) + assert unavailable.value.status_code == 503 + assert not session["rotated"] and not session["minted"] and not session["revoked"] + restored = await session["store"].take_refresh("unit-refresh", "unit-client") + assert restored["family"] == "unit-family" + assert restored["upstream_refresh"] == "unit-upstream-old" + assert restored["verified_email"] == EMAIL + assert not session["store"].owed + + +async def test_later_outage_keeps_an_already_rotated_upstream_credential( + people, session, monkeypatch +): + async def rotated_then_unavailable(current, refresh): + session["rotated"].append(current["upstream_refresh"]) + authz.forget_ended("identity", "alice") + people.unavailable = True + return "unit-upstream-next" + + monkeypatch.setattr(oauth, "_ask_identity_provider", rotated_then_unavailable) + await session["store"].save_refresh("unit-refresh", grant()) + with pytest.raises(oauth.OAuthError) as unavailable: + await oauth._refresh_token_grant( + {"client_id": "unit-client", "refresh_token": "unit-refresh"} + ) + assert unavailable.value.status_code == 503 + restored = await session["store"].take_refresh("unit-refresh", "unit-client") + assert restored["upstream_refresh"] == "unit-upstream-next" + assert restored["verified_email"] == EMAIL and restored["family"] == "unit-family" + assert not session["minted"] and not session["store"].owed + + +async def test_n_minus_one_refresh_without_email_uses_current_accounts(session): + original = grant() + del original["verified_email"] + await session["store"].save_refresh("unit-refresh", original) + result = await oauth._refresh_token_grant( + {"client_id": "unit-client", "refresh_token": "unit-refresh"} + ) + assert result["access_token"] == "unit-session-access" + + +@pytest.mark.parametrize("email", ["", EMAIL]) +def test_code_optional_email_round_trip_and_n_minus_one_rewrite(session, email): + original = grant(verified_email=email) + encoded = oauth_store._record(original, oauth_store._CODE_FIELDS, spent=False) + assert encoded["verified_email"] == email + assert oauth_store._code(encoded)["verified_email"] == email + legacy = {k: v for k, v in encoded.items() if k != "verified_email"} + assert oauth_store._code(legacy)["verified_email"] == "" + assert legacy["nonce"] == encoded["nonce"] + + +async def test_native_refresh_row_and_parser_keep_the_optional_email(session, monkeypatch): + row = oauth_store._refresh_row("unit-id", grant(), 1) + assert row["verified_email"] == EMAIL + + class Rows: + async def query_one(self, *args, **kwargs): + return dict(row) + + store = oauth_store.ClickHouseStore() + monkeypatch.setattr(store, "_ch", lambda: Rows()) + parsed = await store._refresh_grant("unit-id") + assert parsed["verified_email"] == EMAIL + del row["verified_email"] + assert (await store._refresh_grant("unit-id"))["verified_email"] == "" + + +async def test_refresh_put_back_preserves_held_cas_values_and_email(session): + await session["store"].save_refresh("unit-refresh", grant()) + held = await session["store"].take_refresh("unit-refresh", "unit-client") + raw = dict(held["held"]) + await session["store"].save_refresh("unit-refresh", held) + assert held["verified_email"] == EMAIL + assert all(isinstance(value, str) for value in raw.values()) + restored = await session["store"].take_refresh("unit-refresh", "unit-client") + assert restored["verified_email"] == EMAIL and restored["family"] == held["family"] + + +async def test_device_sign_in_keeps_trusted_email_in_real_cas_record(session): + user_code = device.normalize_user_code("BCDF-GHJK") + record = { + "id": "unit-device", + "client_id": device.DEVICE_CLIENT_ID, + "subject": "", + "org": "", + "namespaces": [], + "roles": [], + "exp": int(time.time()) + 600, + "family": "", + "upstream_refresh": "", + "user_code": device._user_code_id(user_code), + "status": "pending", + "address": "unit-address", + "ticket": "", + } + await session["store"].save_device(record) + response = await device.signed_in(user_code, IDENTITY, "unit-upstream", verified_email=EMAIL) + assert response.status_code == 200 + assert b"Approve sign-in?" in response.body + saved = await session["store"].device("unit-device") + assert saved["verified_email"] == EMAIL + raw = json.loads(session["store"].keeper["device:unit-device"]) + assert raw["verified_email"] == EMAIL + + +@pytest.mark.parametrize("outage", [False, True]) +async def test_consent_rechecks_before_spending_the_ticket_or_writing_a_code( + people, session, outage +): + ticket = "unit-approval" + pending = { + "kind": "connect", + "client_id": "unit-client", + "redirect_uri": "https://example.test/callback", + "code_challenge": "challenge", + "mcp_state": "unit-state", + "identity": list(IDENTITY), + "verified_email": EMAIL, + "since": int(time.time()), + "upstream_refresh": oauth.seal(oauth.UPSTREAM_PURPOSE, "unit-upstream"), + } + await session["store"].save_pending(oauth_store.token_id(ticket), pending) + people.users.clear() + people.unavailable = outage + response = TestClient(oauth_app()).post( + "/oauth/authorize/approve", + data={"ticket": ticket, "decision": "approve"}, + follow_redirects=False, + ) + assert response.status_code == (503 if outage else 400) + assert await session["store"].pending(oauth_store.token_id(ticket)) == pending + assert not session["store"].records("code") and not session["minted"] + if outage: + assert response.headers["Retry-After"] == "5" + + +async def test_code_policy_outage_restores_the_same_code_and_verified_email(people, session): + verifier = "unit-code-verifier" * 3 + challenge = ( + base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + ) + original = grant(code_challenge=challenge) + await session["store"].save_code("unit-code", original) + people.unavailable = True + with pytest.raises(oauth.OAuthError) as unavailable: + await oauth._authorization_code_grant( + { + "client_id": "unit-client", + "code": "unit-code", + "code_verifier": verifier, + "redirect_uri": original["redirect_uri"], + } + ) + assert unavailable.value.status_code == 503 + restored = await session["store"].take_code("unit-code", "unit-client", lambda _: True) + assert restored["verified_email"] == EMAIL + assert restored["upstream_refresh"] == original["upstream_refresh"] + assert restored["family"] == original["family"] + assert not session["minted"] and not session["store"].owed + + +def approved_device(*, status="approved", ticket="unit-approval"): + return { + "id": oauth_store.token_id("unit-device-code"), + "client_id": device.DEVICE_CLIENT_ID, + "subject": "alice", + "org": "acme", + "namespaces": ["private"], + "roles": ["writer"], + "exp": int(time.time()) + 600, + "family": "unit-family", + "upstream_refresh": "unit-upstream", + "verified_email": EMAIL, + "user_code": device._user_code_id("BCDFGHJK"), + "status": status, + "address": "unit-address", + "ticket": oauth_store.token_id(ticket), + } + + +@pytest.mark.parametrize("outage", [False, True]) +async def test_device_approval_rechecks_before_live_grant_and_keeps_retryable_state( + people, session, outage +): + original = approved_device(status="pending") + await session["store"].save_device(original) + before = session["store"].keeper["device:" + original["id"]] + people.users.clear() + people.unavailable = outage + response = TestClient(oauth_app()).post( + "/oauth/device/approve", + data={"user_code": "BCDFGHJK", "ticket": "unit-approval", "decision": "approve"}, + ) + assert response.status_code == (503 if outage else 200) + saved = await session["store"].device(original["id"]) + assert saved["status"] == ("pending" if outage else "denied") + assert not session["minted"] + if outage: + assert response.headers["Retry-After"] == "5" + assert session["store"].keeper["device:" + original["id"]] == before + assert not session["revoked"] + else: + assert saved["family"] == "" and saved["upstream_refresh"] == "" + + +async def test_device_grant_outage_puts_back_same_approved_cas_record(people, session): + original = approved_device() + await session["store"].save_device(original) + people.unavailable = True + with pytest.raises(oauth.OAuthError) as unavailable: + await device.device_code_grant( + {"client_id": device.DEVICE_CLIENT_ID, "device_code": "unit-device-code"} + ) + assert unavailable.value.status_code == 503 + restored = await session["store"].take_device("unit-device-code") + assert restored["status"] == "approved" + assert restored["verified_email"] == EMAIL + assert restored["family"] == original["family"] + assert restored["upstream_refresh"] == original["upstream_refresh"] + assert not session["minted"] and not session["revoked"] + + +@pytest.mark.parametrize("kind", ["code", "device"]) +async def test_previous_writer_drops_email_proof_without_changing_raw_cas_ownership( + people, session, kind +): + if kind == "code": + await session["store"].save_code("unit-code", grant()) + key = "code:" + oauth_store.token_id("unit-code") + else: + original = approved_device() + await session["store"].save_device(original) + key = "device:" + original["id"] + old = session["store"].keeper[key] + legacy = {name: value for name, value in json.loads(old).items() if name != "verified_email"} + legacy["nonce"] = "unit-previous-writer-nonce" + rewritten = json.dumps(legacy) + assert await session["store"]._swap(key, old, rewritten, int(legacy["exp"])) == rewritten + if kind == "code": + parsed = await session["store"].take_code("unit-code", "unit-client", lambda _: True) + else: + parsed = await session["store"].take_device("unit-device-code") + assert parsed["verified_email"] == "" + assert list(parsed["held"].keys()) == [key] + assert json.loads(parsed["held"][key])["spent"] is True + people.accounts[:] = [("acme", "alice", EMAIL, False)] + assert await scim.refuses_subject( + parsed["subject"], org=parsed["org"], verified_email=parsed["verified_email"] + ) + assert not session["minted"] diff --git a/tests/unit/test_sharing.py b/tests/unit/test_sharing.py index 891e652df2..52fbb30581 100644 --- a/tests/unit/test_sharing.py +++ b/tests/unit/test_sharing.py @@ -7,6 +7,7 @@ import asyncio from datetime import datetime from typing import Any +from unittest.mock import AsyncMock import pytest @@ -711,7 +712,7 @@ def test_an_erasure_or_export_of_no_user_is_refused(value: str) -> None: ErasureRequestInput(**scope) -def test_an_end_user_token_is_served_only_where_end_users_are( +async def test_an_end_user_token_is_served_only_where_end_users_are( monkeypatch: pytest.MonkeyPatch, ) -> None: """The op endpoint and downloads serve it; `/v1/me/*`, the admin routes and @@ -720,14 +721,16 @@ def test_an_end_user_token_is_served_only_where_end_users_are( from server.auth import bearer - monkeypatch.setattr(bearer, "verify_bearer", lambda _token: _user("alice")) + monkeypatch.setattr( + bearer, "verify_bearer", AsyncMock(side_effect=lambda _token: _user("alice")) + ) with pytest.raises(HTTPException) as refused: - bearer.verify_bearer_header("Bearer eu") + (await bearer.verify_bearer_header("Bearer eu")) assert refused.value.status_code == 403 - assert bearer.verify_bearer_header("Bearer eu", acts_for_end_users=True) is not None + assert (await bearer.verify_bearer_header("Bearer eu", acts_for_end_users=True)) is not None -def test_the_op_endpoint_runs_an_end_user_token_as_its_user( +async def test_the_op_endpoint_runs_an_end_user_token_as_its_user( monkeypatch: pytest.MonkeyPatch, ) -> None: from types import SimpleNamespace @@ -735,8 +738,10 @@ def test_the_op_endpoint_runs_an_end_user_token_as_its_user( from server.auth import bearer from server.transport import http - monkeypatch.setattr(bearer, "verify_bearer", lambda _token: _user("alice", "helper")) - auth = http._extract_auth(SimpleNamespace(headers={"Authorization": "Bearer eu"})) # type: ignore[arg-type] + monkeypatch.setattr( + bearer, "verify_bearer", AsyncMock(side_effect=lambda _token: _user("alice", "helper")) + ) + auth = await http._extract_auth(SimpleNamespace(headers={"Authorization": "Bearer eu"})) # type: ignore[arg-type] assert auth is not None and (auth.subject, auth.agent_principal) == ( "user:alice", "agent:helper", @@ -850,7 +855,7 @@ async def _run(subject: str) -> None: asyncio.run(_run("group:gone")) -def test_mcp_refuses_an_end_user_token_as_not_allowed_not_as_invalid( +async def test_mcp_refuses_an_end_user_token_as_not_allowed_not_as_invalid( monkeypatch: pytest.MonkeyPatch, ) -> None: from types import SimpleNamespace @@ -863,7 +868,7 @@ def test_mcp_refuses_an_end_user_token_as_not_allowed_not_as_invalid( monkeypatch.setattr(jwt, "verify_token", lambda _token: _user("alice")) request = SimpleNamespace(headers={"authorization": "Bearer eu"}) with pytest.raises(PermissionError) as refused: - mcp_auth._verify_bearer(request) # type: ignore[arg-type] + (await mcp_auth._verify_bearer(request)) # type: ignore[arg-type] response = Response() answer = mcp_auth._not_on_mcp(response, {"id": 7}, refused.value) assert response.status_code == 403 and answer["id"] == 7 diff --git a/tests/unit/test_task_watch.py b/tests/unit/test_task_watch.py index 487bc33343..ec20dbd47a 100644 --- a/tests/unit/test_task_watch.py +++ b/tests/unit/test_task_watch.py @@ -7,6 +7,7 @@ import json from datetime import UTC, datetime from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import FastAPI @@ -234,7 +235,9 @@ async def test_a_present_bearer_is_honoured_even_when_auth_is_off( gateway=False, has_role=lambda role: False, ) - monkeypatch.setattr("server.auth.bearer.verify_token", lambda tok: principal) + monkeypatch.setattr( + "server.auth.bearer.authenticate_token", AsyncMock(side_effect=lambda tok: principal) + ) async def fetch_inbox(org: str, inbox: str) -> dict[str, Any] | None: return {"org": org, "inbox": inbox, "owner_namespace": "ns"} diff --git a/tests/unit/test_token_mint.py b/tests/unit/test_token_mint.py index 42f56791e0..309e970ae0 100644 --- a/tests/unit/test_token_mint.py +++ b/tests/unit/test_token_mint.py @@ -11,6 +11,7 @@ from datetime import timedelta from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException @@ -72,7 +73,9 @@ async def account_active(org: str, creator: str) -> bool: async def _setup(monkeypatch: pytest.MonkeyPatch, caller: AuthInfo, body: dict) -> dict[str, Any]: - monkeypatch.setattr("server.auth.bearer.verify_bearer", lambda token: caller) + monkeypatch.setattr( + "server.auth.bearer.verify_bearer", AsyncMock(side_effect=lambda token: caller) + ) request: Any = SimpleNamespace(headers={"Authorization": "Bearer t"}, state=SimpleNamespace()) return await me.me_service_accounts_create(request, body) diff --git a/tests/unit/test_trusted_caller.py b/tests/unit/test_trusted_caller.py index 98041c0f7c..0e9c9f77bf 100644 --- a/tests/unit/test_trusted_caller.py +++ b/tests/unit/test_trusted_caller.py @@ -18,7 +18,7 @@ from collections.abc import Awaitable, Callable from pathlib import Path from typing import TYPE_CHECKING, Any -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -53,7 +53,7 @@ def _auth(subject: str = "sa:gateway"): ) -def test_extract_auth_honors_obo_header_for_trusted_caller() -> None: +async def test_extract_auth_honors_obo_header_for_trusted_caller() -> None: """sa:gateway is on LOOM_TRUSTED_CALLERS → header replaces subject.""" from server.transport import http as http_mod @@ -65,17 +65,17 @@ def test_extract_auth_honors_obo_header_for_trusted_caller() -> None: ) with ( - patch.object(bearer_mod, "verify_token", return_value=_auth("sa:gateway")), + patch.object(bearer_mod, "authenticate_token", return_value=_auth("sa:gateway")), patch.object(http_mod.settings, "trusted_callers", "sa:gateway,sa:other"), ): - out = http_mod._extract_auth(req) # type: ignore[arg-type] + out = await http_mod._extract_auth(req) # type: ignore[arg-type] assert out is not None assert out.subject == "bob@example.com" assert out.impersonated_by == "sa:gateway" -def test_extract_auth_ignores_obo_when_caller_not_trusted() -> None: +async def test_extract_auth_ignores_obo_when_caller_not_trusted() -> None: """sa:imposter not in LOOM_TRUSTED_CALLERS → header is ignored.""" from server.transport import http as http_mod @@ -87,17 +87,17 @@ def test_extract_auth_ignores_obo_when_caller_not_trusted() -> None: ) with ( - patch.object(bearer_mod, "verify_token", return_value=_auth("sa:imposter")), + patch.object(bearer_mod, "authenticate_token", return_value=_auth("sa:imposter")), patch.object(http_mod.settings, "trusted_callers", "sa:gateway"), ): - out = http_mod._extract_auth(req) # type: ignore[arg-type] + out = await http_mod._extract_auth(req) # type: ignore[arg-type] assert out is not None assert out.subject == "sa:imposter" # unchanged assert out.impersonated_by == "" -def test_extract_auth_ignores_obo_when_no_trusted_callers_configured() -> None: +async def test_extract_auth_ignores_obo_when_no_trusted_callers_configured() -> None: """Empty LOOM_TRUSTED_CALLERS → impersonation feature is off entirely.""" from server.transport import http as http_mod @@ -109,17 +109,17 @@ def test_extract_auth_ignores_obo_when_no_trusted_callers_configured() -> None: ) with ( - patch.object(bearer_mod, "verify_token", return_value=_auth("sa:gateway")), + patch.object(bearer_mod, "authenticate_token", return_value=_auth("sa:gateway")), patch.object(http_mod.settings, "trusted_callers", ""), ): - out = http_mod._extract_auth(req) # type: ignore[arg-type] + out = await http_mod._extract_auth(req) # type: ignore[arg-type] assert out is not None assert out.subject == "sa:gateway" assert out.impersonated_by == "" -def test_extract_auth_trusted_caller_without_obo_header_is_noop() -> None: +async def test_extract_auth_trusted_caller_without_obo_header_is_noop() -> None: """Trusted caller making a normal request (no impersonation) — works as today; subject stays the JWT's actual sub.""" from server.transport import http as http_mod @@ -127,10 +127,10 @@ def test_extract_auth_trusted_caller_without_obo_header_is_noop() -> None: req = _FakeRequest({"Authorization": "Bearer fake-token"}) with ( - patch.object(bearer_mod, "verify_token", return_value=_auth("sa:gateway")), + patch.object(bearer_mod, "authenticate_token", return_value=_auth("sa:gateway")), patch.object(http_mod.settings, "trusted_callers", "sa:gateway"), ): - out = http_mod._extract_auth(req) # type: ignore[arg-type] + out = await http_mod._extract_auth(req) # type: ignore[arg-type] assert out is not None assert out.subject == "sa:gateway" @@ -452,7 +452,9 @@ def _trusted(monkeypatch: pytest.MonkeyPatch) -> object: async def grant_map(org: str, subject: str) -> grant_cache.GrantMap: return grant_cache.GrantMap(ranks={}, held=(subject.lower(),)) - monkeypatch.setattr(bearer_mod, "verify_token", lambda *a, **k: _auth("sa:gateway")) + monkeypatch.setattr( + bearer_mod, "authenticate_token", AsyncMock(side_effect=lambda *a, **k: _auth("sa:gateway")) + ) monkeypatch.setattr(http_mod.settings, "trusted_callers", "sa:gateway") monkeypatch.setattr(grant_cache, "grant_map", grant_map) return TestClient(app) From 2397af549f22e054098da06184d293a5c4246d23 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:41:21 +0000 Subject: [PATCH 110/181] Describe the fixture owner's recorded ownership digest Signed-off-by: zlareb1 --- docs/benchmarks/results.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 0f9bd094b1..00d4ca919b 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -563,7 +563,7 @@ diff <(jq -S . /tmp/summary.json) <(jq -S . bench/code_ask_cache_threshold/summa ``` Embedding APIs are not bit-stable across provider revisions: expect cosines to agree to a -few thousandths. A threshold is robust only when its margin exceeds that drift. +few thousandths. A threshold's margin must exceed that drift. **With the guard on** (`sweep_with_guard`: paraphrase hits of 42 and false hits of 65 different pairs): @@ -1260,9 +1260,11 @@ PYTHONPATH=$PWD .venv/bin/python bench/cell_trigger_gate.py \ The fixture owner supplies the exact source, launch, configuration and ownership receipts, excludes competing controllers and workers, and stops its processes before deleting its -exact scratch databases. The output path must not exist. The driver verifies those -receipts and the live fixture before native setup and every arm, restores the org triggers -within its cleanup bound and emits a final verdict only after cleanup and artifact sync. +exact scratch databases. The output path must not exist. The driver verifies source, +launch and configuration identities and the live fixture before native setup and every +arm, restores the org triggers within its cleanup bound and emits a final verdict only +after cleanup and artifact sync. It records the ownership-receipt digest supplied by +the fixture owner. Source tests and the diagnostic figures above are not evidence that this workload passes. ## Reference computations From a49d33b48eb66d66c2aa42c91173eea0f901db61 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:46:08 +0000 Subject: [PATCH 111/181] Document SCIM person checks and expose generated schema on CI mismatch Describe complete identity decisions and session metadata, retain the migration journal, and export the actual CI schema for review without weakening the schema check. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 11 +++ docs/architecture/cells.md | 96 ++++++++++++------- docs/architecture/security.md | 44 +++++++-- docs/architecture/storage-layout.md | 4 +- docs/operations/sso.md | 29 +++++- docs/reference/configuration.md | 6 +- docs/reference/errors.md | 2 +- docs/reference/mcp.md | 6 +- loom.example.yaml | 13 ++- schemas/loom-settings.schema.json | 4 +- server/erasure/store/census.py | 10 +- server/settings/catalog.py | 12 ++- .../integration/test_scim_erasure_live_ch.py | 56 +++++++---- tests/unit/test_schema_baseline.py | 1 + 14 files changed, 205 insertions(+), 89 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f3ed6e347..ea4c559ab1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -315,7 +315,18 @@ jobs: - name: Ledger is a subset of the repo (strict dry-run) run: python -m server.core.schema.migrate --dry-run --strict - name: Verify schemas/schema.sql matches what migrations produce + id: schema_current run: make schema-check + - name: Export the generated schema after a mismatch + if: failure() && steps.schema_current.outcome == 'failure' + run: python -m scripts.ops.dump_schema --out "$RUNNER_TEMP/schema.sql" + - name: Upload the generated schema after a mismatch + if: failure() && steps.schema_current.outcome == 'failure' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: generated-schema + path: ${{ runner.temp }}/schema.sql + if-no-files-found: error ts-typecheck: name: tsc (frontend) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index c7cabd71fd..064e14b301 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -272,42 +272,66 @@ require assignment and to deactivate on unassignment. Each cell keeps its SCIM u deactivations and tombstones exactly as a single deployment does ([SSO and SCIM](../operations/sso.md)); the router holds no identity. -A cell in `enforced` with SCIM on requires SCIM: a person acts in the cell only while -its SCIM holds an active row for them. The check is on the person's account, not on how -a subject is spelled: `scim.refuses_subject(subject)` takes the subject's own email -spellings and the contact and verified emails of the active accounts it owns, password -accounts included, and refuses if any of them has no active row (the row whose id is -`scim.user_id(email)`; an inactive or deleted row with that id always refuses, and an -email with no row of its own passes only as a work address of an active row whose -primary email is one of the subject's own spellings). A principal outside SCIM is one of -a closed list: a bot (a subject whose `sa:` before any further `:` names a -`loom.machine_identities` row: the bot and its agents, as `check_acted_for` parses it), -a machine or workload subject, an end user (`user:`). Any other `sa:` subject, a -personal service account, is checked through its creator (`service_accounts.sa_creator`) -and every subject its row's `lineage` lists, so acting for it is acting for the person -who made it; every other subject on these paths is a person, and is refused unless it -has an email spelling or owns an account with one. A SCIM read that fails raises -`SignInUnavailable`, as `scim.deactivated` does, never an `OAuthError`, so a refresh -puts its token back and no session ends on an outage. Every path that admits a person -uses it: - -- an SSO sign-in, before the account is found or made (JIT), with the email the identity - provider verified; -- every refresh of a session (`_refresh_session`), with the verified email the grant - stores, or, for a grant without one, the subject's accounts; -- an act on a person's behalf (`authz.check_acted_for`), whatever the subject's - spelling; -- a gateway token's subject, checked when verified and remembered as verification's - other answers are; -- the creator of a delegated token or personal service account (`authz._check_creator`), - cached as the acted-for check is, so a token made by a person never provisioned is - refused once the cell is `enforced`. - -SCIM on already closes password sign-in and self-signup (`oauth_login_provider=oidc`), -and the device flow signs in through SSO, so through the same check. Principals without -an email are outside SCIM and deprovisioned by whoever made them: bots and minted tokens -by an operator, end-user issuers' users by the service that signs for them, workload -trust by its rule. +A cell in `enforced` with SCIM on admits a person only while its SCIM data +qualifies that person's complete identity graph. `scim.refuses_subject` checks the +primary subject, token creator, every parent and any acted-for subject together, in +the concrete target org before grants or placement. A wildcard token supplies no +guessed org; the concrete operation checks the complete graph again. + +A person's signed email and the verified email from their OIDC sign-in are positive +ownership witnesses. In the target org, the check resolves one normalized owner from +an active account's owner or its verified contact pair (`VERIFIED_EMAIL_SQL`). Two +different owners refuse. An email-shaped canonical owner is both mandatory and a +positive witness. Every contact on every active account of that owner is mandatory, +including an unlinked password contact; that unverified contact cannot supply a +positive witness or resolve another owner. At least one positive witness is required. +Every mandatory address needs its own active, undeleted `scim.user_id(email)` row. +Only a missing own row may pass as a work address of an active primary row belonging +to this person's positive witnesses. An inactive or deleted own row always refuses. +One person's verified email never qualifies a different root or service-account +ancestor. + +Exemptions are closed: an active registered bot and its agents, including machine or +workload exchanges backed by that registration, and a signed end user whose issuer +and end-user metadata were verified. A prefix, an admin role or the absence of email +creates no exemption. Other `sa:` subjects use the current personal service-account +row: it must exist, remain unrevoked and unexpired, and have a creator. Its creator +and every persisted lineage subject must qualify, even when grant inheritance is +disabled. This security read bypasses the grant-inheritance metadata cache. Cycles +refuse; a shared ancestor completed on another path is checked once. + +One decision allows at most 1,024 supplied roots, distinct subjects, examined lineage +edges, account rows and distinct emails in each dimension. Repeated root entries +count before deduplication, and repeated lineage edges count as examined work. Input +and serialized query parameters each fit 65,536 bytes; account and ancestry bytes +transferred across the decision share that bound. SQL returns overflow counts and +bounded projections rather than accepting a truncated prefix. Unsupported sizes +refuse before admission. The traversal costs O(S + E) for S subjects and E examined +edges, plus the bounded account data; resolving an owner still scans that org's +account rows. + +A complete cold decision has one finite 30-second deadline, reduced by a smaller +positive `LOOM_CH_TIMEOUT_READ_S`. Setting that field to zero leaves general reads +unbounded but does not remove the SCIM deadline. Queries use the remaining decision +budget through the normal ClickHouse statement limits; a tighter scoped or query-class +limit still applies. A read failure or deadline raises `SignInUnavailable`, which +becomes retryable 503 rather than deactivation. The refresh path checks this policy +before rotating the provider's credential and puts the same token and family back +on an outage. Positive refusal and an unavailable read are different outcomes. + +Only a completed whole-graph result is memoized, in authorization's existing final +memo. Its key includes deployment and cell identity, concrete org, exact primary +subject, normalized verified email and canonical additional roots. No inner subject +or ancestry cache supplies a security answer. `LOOM_GRANT_CACHE_TTL_SECONDS` is counted +from the complete read's start, not its finish or a cache hit. Identity, account, +link, machine and lineage writes invalidate through the existing forget hook before +the attempt and in `finally`, so an overtaken read cannot install its stale result. +Across processes, an already-running decision can observe a change within at most +`max(grant_cache_ttl_seconds, decision_s)`, where `decision_s` is the finite complete +decision budget. This is not a globally atomic identity snapshot; ordinary revocation +checks retain their separate rules. Standalone deployments and cells without the +required policy retain deactivation checking without installing a positive-provisioning +permit. The global cell gate still refuses unknown or unread identity. **Break-glass.** Every cell holds a key-proof platform-admin bot (`loom bot create --platform-admin`), which has no email and which SCIM never revokes, diff --git a/docs/architecture/security.md b/docs/architecture/security.md index cb78320c69..a19e8e8b1f 100644 --- a/docs/architecture/security.md +++ b/docs/architecture/security.md @@ -390,8 +390,12 @@ leaves its namespace out is refused, never run on the install's. The one listed `memory.delete`, acts on the key it names, in the install's namespace, under any agent of the same user, as it does for a named agent. A call with a token is filled with nothing, nor is any call in another mode, so tenant isolation and failing closed are unchanged -wherever a token decides who the caller is. In `trust-network`, passwords are argon2id and -there is no email verification or password reset. In `gateway`, Loom never sees a +wherever a token decides who the caller is. Caller identity and token-management routes +use the deliberate dev identity only when no credential is supplied. A malformed bearer +or another authorization scheme is refused; auth off does not turn it into `you`. +MCP's `open` mode retains its separate [authentication rules](../reference/mcp.md#authentication). +In `trust-network`, passwords are argon2id and there is no email verification or password +reset. In `gateway`, Loom never sees a password. **Agent claims.** An agent account is unclaimed while its password is its claim code @@ -479,12 +483,11 @@ HTTP to act as another principal. The header's subject becomes the effective sub or deprovisioned, in any case and under any spelling it holds in the target org or the caller's own org (itself, and the handle and email of each active account of it whose email is verified), is not acted for (403), nor is a bot that is revoked outright or disabled, nor -any of its agents (a disable leaves their grants). Of the `sa:` subjects, only an active bot -and its agents are acted for: any other is a token's own subject, whose direct grants belong -to the token and must end with it, and a token ends in ways its grants do not record (an -expiry, a revoke by `jti`, a cutoff on its maker or on itself), so a trusted caller acting -for one would keep reaching what a dead token held. A token minted under a subject without -`sa:` cannot be told from a person's, and is acted for until its subject is revoked +any of its agents (a disable leaves their grants). A personal service-account subject +requires its current row to exist, be unrevoked and unexpired, and name a creator. In an +enforced cell with SCIM on, that creator and every lineage subject must also pass the +complete provisioning check; the inheritance cache supplies no security answer. A token +minted under a subject without `sa:` cannot be told from a person's, and is acted for until its subject is revoked outright. A request that names no org, from a caller whose token is for every org, has no org to read spellings in, so only the subject as sent is checked. An unverified email is not held: revoking a stranger's account that typed someone's email does not refuse acting for @@ -545,6 +548,16 @@ for the same reason. A SCIM user's id is a UUIDv5 of its email, so it is a pseud secret: anyone who knows an address can compute its id. Keying it with the root secret would change every id when that secret rotates, and the provider would find none of its users. +An enforced cell with SCIM on requires active provisioning; absence refuses a person. [Cell provisioning](cells.md#scim-and-sign-in-across-cells) +defines the closed exemptions, verified account ownership, complete ancestry and resource +bounds. `server/auth/jwt.py` (`authenticate_token`) adds that asynchronous decision after +synchronous cryptographic verification. Bearer authentication, `whoami`, health's admin +details, caller identity, dev service-account routes, the SCIM Groups JWT fallback, MCP +POST/session DELETE and tenancy middleware use that boundary, directly or through the +common bearer helper. Health remains public but omits denied or unavailable admin details. +`verify_token` remains synchronous; OAuth revoke uses it only to end an existing credential, +so provisioning refusal or outage cannot prevent logout or grant read/write authority. + Loom stays the authorization server for every client (the dashboard, `loom login`, MCP clients); the provider only authenticates the person. The OAuth server: @@ -576,8 +589,8 @@ and lasts what they asked for (blank 90 days, 0 never). A session a third-party holds is not one: what it mints ends with its access token at the latest, and at once when the session is revoked (the session is one of its `parents`), so disconnecting the client ends its reach. Any other parent bounds its children by its own expiry. Under OIDC each -refresh first spends the provider's refresh token, so a user deactivated there loses their -sessions within the hour. Their machine tokens are not re-checked at the provider: +refresh checks local provisioning before spending the provider's refresh token, so a user +deactivated there loses their sessions within the hour. Their machine tokens are not re-checked at the provider: offboarding, or a session found stolen, revokes them (by subject, or each from the owner's service-account list). A revocation of a token that never expires lasts as long as the table can hold a row (2105). A provider that issues no refresh token gets no Loom refresh token. A @@ -611,6 +624,17 @@ table's key limit is the backstop: past it new sign-ins answer 503 and `/v1/heal client is told to retry, so its retry is not taken for a replay. A refresh token without a family is refused, so its owner signs in again. +The OIDC-verified email is carried explicitly through resolved sign-in, consent, codes, +device approval and refresh grants. `verified_email` is optional session metadata in the +Keeper record and `loom.oauth_state` (`String DEFAULT ''`). A build without the field can +read the record and drop it on rewrite; the receiving build treats omission as empty and +checks the actual subject and accounts, never a guessed email or a verification exemption. +The held compare-and-set value remains the exact raw value read. A local provisioning +outage uses the existing refresh put-back path, preserving the same token, family and +current sealed upstream credential. Account activity and roles are still rechecked, so +stored email metadata cannot revive a released account. These auth rows hold no memory +content; their expiry and erasure classification do not change. + **The IdP round trip is bound to its browser.** What the callback needs (the nonce, the PKCE verifier, where to send the result) travels sealed in an HttpOnly cookie of the browser that started the sign-in, never stored on the server. The callback finishes a diff --git a/docs/architecture/storage-layout.md b/docs/architecture/storage-layout.md index d5ad074948..12a9043bef 100644 --- a/docs/architecture/storage-layout.md +++ b/docs/architecture/storage-layout.md @@ -904,9 +904,9 @@ copies of tables listed here, owned and removed by that run, and erasure reaches | `scim_users` | RMT(version) `id` | Postgres `scim_users` | SoR | no content | 1 | deprovisioning revokes tokens: a stale read keeps access; named in `server/auth/scim.py` | | `sso_links` | RMT(version) `(email, org, namespace)` | Postgres `sso_links` | SoR | no content | 1 | account ownership decided at sign-in by `(email, org, namespace)`; named in `server/auth/sso_links.py`, `server/identity/store/namespace_census.py` | | `principal_groups` | RMT(version) `(org, group_id)` | Postgres `principal_groups` | SoR | step `groups.remove_member` | 1 | group membership expands grants at check time; named in `server/identity/app/scim_groups.py`, `server/erasure/store/steps_principals.py` | -| `oauth_state` | RMT(version) `(record_type, id)`, by `record_type`, TTL | Postgres `oauth_state` | SoR | no content | 1 | registered clients, refresh tokens' grants (written before the token is handed out) and owed revocations (`revocation_owed`); what a sign-in spends once is in `oauth_signin_state`; today's 90-day TTL becomes a sweep; named in `server/auth/oauth_store.py`, `server/auth/revocation.py` | +| `oauth_state` | RMT(version) `(record_type, id)`, by `record_type`, TTL | Postgres `oauth_state` | SoR | no content | 1 | registered clients, refresh grants including optional OIDC-verified email metadata (written before the token is handed out) and owed revocations (`revocation_owed`); what a sign-in spends once is in `oauth_signin_state`; today's 90-day TTL becomes a sweep; named in `server/auth/oauth_store.py`, `server/auth/revocation.py` | | `oauth_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table | -| `oauth_signin_state` | KeeperMap `claim_key` | Postgres `oauth_signin_state` | SoR | no content | 1 | strict compare-and-set per key with an expiry: an `UPDATE … WHERE value = $old`; the KeeperMap expiry becomes `expires_at`, read by a sweep; named in `server/auth/claims.py`, `server/cli/secrets.py` | +| `oauth_signin_state` | KeeperMap `claim_key` | Postgres `oauth_signin_state` | SoR | no content | 1 | strict compare-and-set per key with an expiry and optional verified sign-in email metadata: an `UPDATE … WHERE value = $old`; the KeeperMap expiry becomes `expires_at`, read by a sweep; named in `server/auth/claims.py`, `server/cli/secrets.py` | | `grant_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table (`server/auth/grant_claims.py` is the claim module, whose keys are in `_migration_lock`) | | `namespace_grants` | RMT(version) `(org, namespace, subject)`, by `org` | Postgres `namespace_grants` | SoR | step `grants.revoke` | 1 | every request's role check; a revoke takes effect within the grant cache's invalidation bound; named in `server/identity/store/grants.py`, `server/auth/self_signup.py` | | `namespaces` | RMT(updated_at) `(org, namespace)`, by `org` | Postgres `namespaces` + ClickHouse mirror | SoR (head) | step `personal.retire`; step `collections.forget_owner` | 1 | the registry with its per-namespace settings, read by grants, sign-in and request paths through a cache; the mirror serves ClickHouse worker scans and the SQL surface views; named in `server/auth/self_signup.py`, `server/cli/hygiene.py` | diff --git a/docs/operations/sso.md b/docs/operations/sso.md index 4138c604db..85545f113a 100644 --- a/docs/operations/sso.md +++ b/docs/operations/sso.md @@ -501,6 +501,28 @@ Generate the replacement with the same command used for that deployment's initial token. Both entries on a cell must name that cell, except for unnamed tokens retained by its observed first cell. +On an enforced cell with SCIM on, provision the person before sign-in: an active, +undeleted SCIM row is required before JIT account lookup or creation. Assign the +same people to the cell's sign-in and provisioning applications. Every active +account contact is checked, including password contacts, but an unlinked password +contact cannot prove that its account belongs to the person. Resolve stale or +incorrect account contacts before enforcement. Personal service-account creators +and all lineage subjects must qualify too; neither an admin role nor a minted token +bypasses the rule. See [cell provisioning](../architecture/cells.md#scim-and-sign-in-across-cells) +for the closed machine and signed-end-user exemptions. + +Missing provisioning and unsupported identity sizes refuse authentication or +authorization. A complete decision supports at most 1,024 roots, subjects, lineage +edges, accounts and emails per dimension, and 65,536 bytes for input, serialized +parameters and transferred identity data. These are fixed security limits. Its +complete deadline is at most 30 seconds, reduced by a smaller positive +`LOOM_CH_TIMEOUT_READ_S`; zero does not disable this SCIM-specific bound. An unread +store or deadline answers retryable 503 with `Retry-After: 5`. A refresh checks local +provisioning before contacting the identity provider and puts its token back on an +outage; logout remains available. These admission checks supplement the revocations +below. The cache TTL alone is not the cross-process observation bound; the linked +architecture describes the complete read budget. + Loom keys a SCIM user on an email: of the request's `emails` entries with an address, the one marked primary, else the first work one, else the first; or its `userName` when it sends none (Loom logs a warning when a user is created @@ -670,9 +692,10 @@ by hand. Reactivate at the provider first anyone who should keep Loom. ### Changing a person's email Loom keys a SCIM user on the address and refuses a change of it (400), so a -new address must arrive as a new SCIM user. Until it does, a sign-in with the -new address makes an account no SCIM user reaches, which the old address's -deactivation misses. Change it before the person signs in with the new one: +new address must arrive as a new SCIM user. An enforced cell with SCIM on refuses +the address until it is provisioned. On a standalone deployment, an earlier sign-in +with the address can make an account no SCIM user reaches, which the old address's +deactivation misses. Change it before the person signs in with the address: 1. Remove the person from the app at the provider. That deactivates the old address and revokes the accounts it reaches. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index a31502c63f..e79a0cccec 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -85,7 +85,7 @@ Authentication, tokens and identity providers. | `LOOM_OIDC_JIT_ORG` | `str` | `''` | | Org that JIT-provisioned accounts are created in; required when `oidc_jit_provision` is on. | | `LOOM_OIDC_JIT_EMAIL_DOMAINS` | `str` | `''` | | Comma-separated email domains JIT provisioning admits; empty admits any verified email. | | `LOOM_IDENTITY_PROVIDER_TIMEOUT_S` | `float` | `10.0` | live | Seconds one call to an identity provider may take: OIDC discovery, the token and revocation endpoints, its signing keys, a machine issuer's discovery, and an end-user issuer's JWKS fetch. | -| `LOOM_SCIM_TOKEN` | `str` | *(unset)* | | Bearer tokens (comma-separated, any one accepted) an identity provider presents to deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. With LOOM_CELL set, at most two entries, named for that cell; only the observed first cell also accepts unnamed tokens. Generate one with `loom cell scim-token new --cell NAME`. | +| `LOOM_SCIM_TOKEN` | `str` | *(unset)* | | Bearer tokens (comma-separated, any one accepted) an identity provider presents to deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. With LOOM_CELL set, at most two entries, named for that cell; only the observed first cell also accepts unnamed tokens. Generate one with `loom cell scim-token new --cell NAME`. Enforced cells also require active provisioning for every nonexempt person. | | `LOOM_JWT_SECRET` | `str` | *(set)* | | The root secret, and the one secret a deployment carries: it signs tokens as kid `default` and seals every stored credential and every signing key in loom.signing_keys (`loom signing-key rotate` adds those). Required when auth is on. The default is the checked-in DEV secret anyone can mint with: a deployment with auth on refuses to start with it, except the `make dev` stack (`loom doctor` warns everywhere else). A non-loopback bind refuses an HMAC secret under 32 bytes or an RSA key under 2048 bits. | | `LOOM_JWT_SECRET_PREVIOUS` | `str` | *(unset)* | | The root secret LOOM_JWT_SECRET replaced, while a rotation is under way: credentials and signing keys sealed under it still open (tokens it signed as kid `default` do not verify: a leaked root mints nothing). `loom secrets reseal` rewrites the stored ones under the new root; then remove this. "" = none. | | `LOOM_JWT_KEYRING_REFRESH_SECONDS` | `float` | `30.0` | | Seconds between re-reads of loom.signing_keys in every process (at least 1); `loom signing-key rotate` waits out two of the slowest process's. | @@ -111,7 +111,7 @@ Authentication, tokens and identity providers. | `LOOM_JWT_LEEWAY_SECONDS` | `int` | `30` | | Clock-skew tolerance on iat/exp, seconds; small enough to give replay no useful post-exp window. | | `LOOM_JWT_REVOCATION_CACHE_TTL_SECONDS` | `float` | `30.0` | | Seconds the `loom.revoked_tokens` snapshot is cached; verify checks both jti and sub against it, so a revocation takes effect on other processes within this TTL (at once on the one that wrote it). | | `LOOM_JWT_REVOCATION_MAX_STALE_SECONDS` | `float` | `600.0` | | Oldest revocation snapshot a process still trusts, seconds. Past it (refreshes keep failing) every bearer is refused with a retryable 503. A subject's cached grants are trusted as long when a re-read fails. | -| `LOOM_GRANT_CACHE_TTL_SECONDS` | `float` | `10.0` | | Seconds a subject's grant rows are cached for the namespace gate, which caps a token's claim at the grant it holds there. The backstop: a grant change reaches other processes through the per-org grant watermark (acl_watermark_poll_seconds), and this bounds one whose watermark row was never written. It also bounds how long a SCIM deactivation of an email, or a bot disable, takes to stop a trusted caller acting for it on another process, and how long a bot created there takes to be acted for. | +| `LOOM_GRANT_CACHE_TTL_SECONDS` | `float` | `10.0` | | Seconds a subject's grant rows are cached for the namespace gate, which caps a token's claim at the grant it holds there. The backstop: a grant change reaches other processes through the per-org grant watermark (acl_watermark_poll_seconds), and this bounds one whose watermark row was never written. SCIM and bot final decisions use this TTL from read start. A complete enforced-cell SCIM check has a separate 30-second ceiling, so its cross-process observation bound is the larger of this TTL and that check's budget; a smaller positive read timeout reduces the latter. | | `LOOM_ACL_WATERMARK_POLL_SECONDS` | `float` | `1.0` | | Seconds between each process's reads of the per-org grant watermark (loom.acl_versions): a revoke on one replica is enforced on every other within about this long. | | `LOOM_ACL_WATERMARK_MAX_STALE_SECONDS` | `float` | `10.0` | | Oldest grant watermark a process still trusts, seconds. Past it (polls keep failing) cached grants are not served and every request reads its own. | | `LOOM_AUTH_ATTEMPTS_CONCURRENCY` | `int` | `8` | | Sign-ins, sign-ups and workload-token exchanges one process counts at once (each costs ClickHouse statements, a sign-in a password hash); past it a caller waits up to a second, then gets 429 (an exchange 503), so a burst cannot take the shared connection pool. | @@ -161,7 +161,7 @@ ClickHouse connection, query shaping and vector storage. | `LOOM_CH_KEEPALIVE_TIMEOUT` | `float` | `5.0` | | Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load balancer's idle cutoff (8 s) so loom closes first: a peer-closed pooled TLS connection is never reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). | | `LOOM_CH_READ_YOUR_WRITES` | `bool` | `True` | | A point lookup that MISSES right after a write (get after set, verify, update_by_id, delete) re-reads once with select_sequential_consistency=1. Costs only on a miss; a no-op on single-node MergeTree. | | `LOOM_CH_READ_YOUR_WRITES_WINDOW_S` | `float` | `20.0` | | memory.search on a namespace this process wrote within this many seconds reads with select_sequential_consistency from the start. 0 = off (the empty-pool retry remains the backstop). | -| `LOOM_CH_TIMEOUT_READ_S` | `float` | `30.0` | | Deadline, seconds, for read statements, applied as `max_execution_time` and as a client-side bound that frees the pool slot. 0 = no bound for reads. | +| `LOOM_CH_TIMEOUT_READ_S` | `float` | `30.0` | | Deadline, seconds, for read statements, applied as `max_execution_time` and as a client-side bound that frees the pool slot. 0 = no general read bound. Enforced-cell SCIM decisions still have a complete 30-second ceiling, reduced by a smaller positive value. | | `LOOM_CH_TIMEOUT_WRITE_S` | `float` | `60.0` | | Deadline, seconds, for INSERT and UPDATE statements; 0 = no bound. | | `LOOM_CH_TIMEOUT_DDL_S` | `float` | `300.0` | | Deadline, seconds, for DDL statements (CREATE, ALTER, DROP); 0 = no bound. | | `LOOM_CH_BATCHED_INSERTS` | `bool` | `True` | live | Batch the inserts written on every request into shared parts; false commits them, and the waited async inserts keep their bounds (docs/architecture/clickhouse.md#batched-inserts). | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 618a445e85..4f8aa945e4 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -254,7 +254,7 @@ HTTP 403, raised as `server.core.errors.AuthInsufficientError`. A public 403 tha #### `AUTH_INVALID` -Raised in `server/auth/bearer.py`, `server/auth/delegation.py`, `server/auth/login.py`. Category `auth`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do). +Raised in `server/auth/bearer.py`, `server/auth/delegation.py`, `server/auth/login.py`, `server/transport/tenancy_gate.py`. Category `auth`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do). #### `AUTH_UNAVAILABLE` diff --git a/docs/reference/mcp.md b/docs/reference/mcp.md index 6a2cb65338..22451377ae 100644 --- a/docs/reference/mcp.md +++ b/docs/reference/mcp.md @@ -59,7 +59,7 @@ instructions. | -------------------- | ------------------------------------------------------------- | | `required` (default) | every method except notifications | | `discovery_only` | `tools/call` only; every other method is open | -| `open` | nothing; an invalid bearer is ignored (trusted networks only) | +| `open` | nothing; a cryptographically invalid bearer is ignored (trusted networks only) | A server on a non-loopback `LOOM_BIND_HOST` refuses to start with `open` unless `LOOM_ALLOW_INSECURE=1`. A deployment with auth off (`LOOM_AUTH_REQUIRED=false`) @@ -76,6 +76,10 @@ The bearer is read from the `Authorization` header only. A token passed as `?tok the `/mcp` URL is ignored and the request is unauthenticated: request lines land in proxy and access logs, where a URL token would be written in the clear. +A bearer that verifies but fails required cell provisioning is refused in every mode; +an unavailable provisioning read answers retryable 503. Neither becomes an anonymous +caller. See [cell provisioning](../architecture/cells.md#scim-and-sign-in-across-cells). + ### OAuth Served only with `LOOM_OAUTH_ENABLED=true`. Loom is an OAuth 2.1 authorization server for clients that diff --git a/loom.example.yaml b/loom.example.yaml index 17df27d780..f5aae8b42a 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -93,7 +93,9 @@ settings: # ch_read_your_writes_window_s: 20.0 # Deadline, seconds, for read statements, applied as `max_execution_time` and as a - # client-side bound that frees the pool slot. 0 = no bound for reads. [restart] + # client-side bound that frees the pool slot. 0 = no general read bound. Enforced-cell + # SCIM decisions still have a complete 30-second ceiling, reduced by a smaller + # positive value. [restart] # ch_timeout_read_s: 30.0 # Deadline, seconds, for INSERT and UPDATE statements; 0 = no bound. [restart] @@ -798,10 +800,11 @@ settings: # Seconds a subject's grant rows are cached for the namespace gate, which caps a # token's claim at the grant it holds there. The backstop: a grant change reaches # other processes through the per-org grant watermark (acl_watermark_poll_seconds), - # and this bounds one whose watermark row was never written. It also bounds how long a - # SCIM deactivation of an email, or a bot disable, takes to stop a trusted caller - # acting for it on another process, and how long a bot created there takes to be acted - # for. [restart] + # and this bounds one whose watermark row was never written. SCIM and bot final + # decisions use this TTL from read start. A complete enforced-cell SCIM check has a + # separate 30-second ceiling, so its cross-process observation bound is the larger of + # this TTL and that check's budget; a smaller positive read timeout reduces the + # latter. [restart] # grant_cache_ttl_seconds: 10.0 # Seconds between each process's reads of the per-org grant watermark diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 7e4b5d50a0..5e96379375 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -473,7 +473,7 @@ "ch_timeout_read_s": { "default": 30.0, "type": "number", - "description": "Deadline, seconds, for read statements, applied as `max_execution_time` and as a client-side bound that frees the pool slot. 0 = no bound for reads." + "description": "Deadline, seconds, for read statements, applied as `max_execution_time` and as a client-side bound that frees the pool slot. 0 = no general read bound. Enforced-cell SCIM decisions still have a complete 30-second ceiling, reduced by a smaller positive value." }, "ch_timeout_write_s": { "default": 60.0, @@ -2796,7 +2796,7 @@ "grant_cache_ttl_seconds": { "default": 10.0, "type": "number", - "description": "Seconds a subject's grant rows are cached for the namespace gate, which caps a token's claim at the grant it holds there. The backstop: a grant change reaches other processes through the per-org grant watermark (acl_watermark_poll_seconds), and this bounds one whose watermark row was never written. It also bounds how long a SCIM deactivation of an email, or a bot disable, takes to stop a trusted caller acting for it on another process, and how long a bot created there takes to be acted for." + "description": "Seconds a subject's grant rows are cached for the namespace gate, which caps a token's claim at the grant it holds there. The backstop: a grant change reaches other processes through the per-org grant watermark (acl_watermark_poll_seconds), and this bounds one whose watermark row was never written. SCIM and bot final decisions use this TTL from read start. A complete enforced-cell SCIM check has a separate 30-second ceiling, so its cross-process observation bound is the larger of this TTL and that check's budget; a smaller positive read timeout reduces the latter." }, "hsts_max_age_seconds": { "default": 31536000, diff --git a/server/erasure/store/census.py b/server/erasure/store/census.py index e0d9bf942d..3ac63c0d18 100644 --- a/server/erasure/store/census.py +++ b/server/erasure/store/census.py @@ -673,8 +673,14 @@ def owner(self, scope_type: str) -> str | None: "auth plane: the people the identity provider provisions (email, the emails " "list it sent, userName, active) and the subjects a deactivation revoked; no memory." ), - "loom.oauth_signin_state": "auth plane: sign-in state in Keeper (codes, sessions); no memory.", - "loom.oauth_state": "auth plane: OAuth client registrations and codes; no memory rows.", + "loom.oauth_signin_state": ( + "auth plane: Keeper sign-in state (codes, devices, sessions), including optional " + "verified sign-in email metadata; no memory content." + ), + "loom.oauth_state": ( + "auth plane: OAuth client registrations, refresh grants with optional verified " + "sign-in email metadata, and owed revocations; no memory content." + ), "loom.oauth_claims": "auth plane: no code path writes it; never holds memory content.", "loom.grant_claims": "auth plane: no code path writes it; never holds memory content.", "loom.service_accounts": "auth plane: service-account definitions.", diff --git a/server/settings/catalog.py b/server/settings/catalog.py index 8bb01606f1..f2ec5b3432 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -277,6 +277,7 @@ def _check_default_namespace(self) -> AuthSettings: # deactivate and reactivate people over SCIM 2.0 (/scim/v2). Empty = off; needs auth on, oidc. # With LOOM_CELL set, at most two entries, named for that cell; only the observed first # cell also accepts unnamed tokens. Generate one with `loom cell scim-token new --cell NAME`. + # Enforced cells also require active provisioning for every nonexempt person. scim_token: str = "" @property @@ -426,9 +427,10 @@ def trusted_issuers(self) -> set[str]: # caps a token's claim at the grant it holds there. The backstop: a grant # change reaches other processes through the per-org grant watermark # (acl_watermark_poll_seconds), and this bounds one whose watermark row was - # never written. It also bounds how long a SCIM deactivation of an email, or a - # bot disable, takes to stop a trusted caller acting for it on another process, - # and how long a bot created there takes to be acted for. + # never written. SCIM and bot final decisions use this TTL from read start. + # A complete enforced-cell SCIM check has a separate 30-second ceiling, so + # its cross-process observation bound is the larger of this TTL and that + # check's budget; a smaller positive read timeout reduces the latter. grant_cache_ttl_seconds: float = 10.0 # Seconds between each process's reads of the per-org grant watermark # (loom.acl_versions): a revoke on one replica is enforced on every other @@ -706,7 +708,9 @@ class ClickHouseSettings(BaseModel): # empty-pool retry remains the backstop). ch_read_your_writes_window_s: float = 20.0 # Deadline, seconds, for read statements, applied as `max_execution_time` and as a - # client-side bound that frees the pool slot. 0 = no bound for reads. + # client-side bound that frees the pool slot. 0 = no general read bound. + # Enforced-cell SCIM decisions still have a complete 30-second ceiling, + # reduced by a smaller positive value. ch_timeout_read_s: float = 30.0 # Deadline, seconds, for INSERT and UPDATE statements; 0 = no bound. ch_timeout_write_s: float = 60.0 diff --git a/tests/integration/test_scim_erasure_live_ch.py b/tests/integration/test_scim_erasure_live_ch.py index e21fb1d73f..1ebc8758ab 100644 --- a/tests/integration/test_scim_erasure_live_ch.py +++ b/tests/integration/test_scim_erasure_live_ch.py @@ -35,8 +35,11 @@ def directory_row(org: str, namespace: str, owner: str, *, ordinary: bool) -> dict[str, Any]: row = registry_row( NamespaceRegisterInput( - org=org, namespace=namespace, owner_email=owner, - kind="memory" if ordinary else None, description="directory description", + org=org, + namespace=namespace, + owner_email=owner, + kind="memory" if ordinary else None, + description="directory description", ), None, datetime.now(UTC).replace(microsecond=0), @@ -58,9 +61,7 @@ async def stored_row(client: ch.CHClient, table: str, org: str, namespace: str) async def cold_decision(auth: AuthInfo, expected_accounts: int, refused: bool) -> None: - extra = scim_people._roots( - auth.subject, "", (*auth.parent_subjects, auth.creator), auth.org - ) + extra = scim_people._roots(auth.subject, "", (*auth.parent_subjects, auth.creator), auth.org) seconds = scim_people._decision_seconds() decision = scim_people._Decision(auth.org, auth.subject, "", scim_people._now() + seconds) with ch.own_write_reads(), ch.query_tag("test.scim_erasure.cold"): @@ -100,7 +101,8 @@ async def test_real_erasure_rewrite_rechecks_a_native_child_after_account_count_ # The foreign personal-name row is an operator-registerable memory row. foreign_row = directory_row(other_org, target, owner, ordinary=kind == "personal") await client.insert( - "loom.namespaces", [*ordinary, target_row, foreign_row], + "loom.namespaces", + [*ordinary, target_row, foreign_row], settings={"async_insert": 0, "wait_for_async_insert": 1}, ) if kind == "collection": @@ -111,12 +113,12 @@ async def test_real_erasure_rewrite_rechecks_a_native_child_after_account_count_ before = await stored_row(client, "loom.namespaces", org, target) foreign_before = await stored_row(client, "loom.namespaces", other_org, target) tree_before = ( - await stored_row(client, "loom.collections", org, target) - if kind == "collection" else None + await stored_row(client, "loom.collections", org, target) if kind == "collection" else None ) foreign_tree_before = ( await stored_row(client, "loom.collections", other_org, target) - if kind == "collection" else None + if kind == "collection" + else None ) _, parsed, _ = scim_routes.user_name_and_email( {"userName": owner, "emails": [{"value": owner, "type": "work", "primary": True}]} @@ -159,17 +161,28 @@ async def read_one(sql: str, params: dict[str, Any] | None = None, **kwargs: Any assert _PARENT_POLICY.__module__ == "server.auth.delegation" assert _PARENT_POLICY.__name__ == "_refuse_a_revoked_parent" parent = AuthInfo( - subject=owner, org=org, namespaces=(), roles=(Role.VIEWER,), - is_service_account=False, impersonated_by="https://issuer.example.test", - read_scope="own_user", read_user_id=user_id, - token_id="parent-" + people.run, issued_at=int(time.time()), + subject=owner, + org=org, + namespaces=(), + roles=(Role.VIEWER,), + is_service_account=False, + impersonated_by="https://issuer.example.test", + read_scope="own_user", + read_user_id=user_id, + token_id="parent-" + people.run, + issued_at=int(time.time()), ) child = await delegation.delegate( - parent, org=org, requested={"*": None}, scoped=False, - ttl_seconds=300, is_service_account=True, + parent, + org=org, + requested={"*": None}, + scoped=False, + ttl_seconds=300, + is_service_account=True, ) expected_keys = sorted( - key for subject in (owner, parent.token_id) + key + for subject in (owner, parent.token_id) for key in (subject, revocation.CUTOFF_PREFIX + subject, revocation.SCIM_PREFIX + subject) ) assert revocation_reads == [({"keys": expected_keys}, [])] @@ -286,7 +299,10 @@ async def execute(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) ] if kind == "collection": claim_keys.append("collections:" + json.dumps(org)) - assert await client.query_all( - "SELECT name FROM loom._migration_lock WHERE name IN {names:Array(String)}", - params={"names": claim_keys}, - ) == [] + assert ( + await client.query_all( + "SELECT name FROM loom._migration_lock WHERE name IN {names:Array(String)}", + params={"names": claim_keys}, + ) + == [] + ) diff --git a/tests/unit/test_schema_baseline.py b/tests/unit/test_schema_baseline.py index 04c56e48a4..5d02a9f40f 100644 --- a/tests/unit/test_schema_baseline.py +++ b/tests/unit/test_schema_baseline.py @@ -156,6 +156,7 @@ def test_the_journal_files() -> None: "098_knowledge_mirrors.sql", "099_document_access_lists.sql", "100_knowledge_source_listings.sql", + "101_oauth_verified_email.sql", ] From 55db6b183bff12c3fa0bf47550da73769257b7e5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:50:08 +0000 Subject: [PATCH 112/181] Name the cell actor in placement fence tests Signed-off-by: zlareb1 --- tests/integration/test_cell_control_fence.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index 2e45540fb4..aa1c33cedf 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -468,7 +468,7 @@ async def wait_for_lock(key): "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) # The placement has its org lock and is queued at the cell row before claim. for _ in range(100): @@ -502,7 +502,7 @@ async def wait_for_lock(key): "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) await wait_for_lock(directory._PLACE_LOCK) await cell_control.backout(_owner(local_admin), call, original) @@ -661,7 +661,9 @@ async def call(path, body): ) # Actually attempt placement before the acknowledged close returns to cleanup. try: - await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + await directory.place( + "between-close-and-cleanup", "first", app.placement_refusal, actor="cell:first" + ) except directory.PlacementRefused as exc: placement_error = str(exc) return answer From 9f84a2b39306b8b71dbd9c8ff3918a55144c4389 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:50:41 +0000 Subject: [PATCH 113/181] Exclude certified retirement history from participant reads Signed-off-by: zlareb1 --- .../insights/store/provider_participants.py | 22 +++++++++++++------ tests/unit/snapshots/settings_defaults.json | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index faa8bc1d54..567aa5e676 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -20,12 +20,24 @@ # Heartbeat age is diagnostic; it never establishes permanent retirement. STALE_AFTER_S = 60 +# This conservative certificate subset never hides unknown retirement evidence. +# Bounded ASCII PIDs also exclude proofs the Python integer parser must refuse. +_CERTIFIED_RETIRED = ( + "state = 'retired' AND isValidUTF8(host_proof) " + "AND length(splitByChar('|', host_proof)) = 9 " + "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " + "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " + "AND length(arrayElement(splitByChar('|', host_proof), 5)) <= 20 " + "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " + "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$')" +) + # `expired` is decided on the server's clock, like the leases. PARTICIPANTS_SQL = ( "SELECT participant_id, transition_id, state, version, heartbeat_at, " "host_proof, " "heartbeat_at + toIntervalSecond(stale_after_s) < now64(3) AS expired " - f"FROM {TABLE} FINAL" + f"FROM {TABLE} FINAL WHERE NOT ({_CERTIFIED_RETIRED})" ) ONE_SQL = ( "SELECT state, version, " @@ -64,12 +76,8 @@ def running_sql(*, inserts: bool = False) -> str: # Retired participants whose last row is older than this are deleted. PRUNE_AFTER_H = 24 _PRUNE_IDS_SQL = ( - f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL WHERE state = 'retired' " - "AND length(splitByChar('|', host_proof)) = 9 " - "AND arrayElement(splitByChar('|', host_proof), 9) = 'cell-retirement-v1' " - "AND arrayAll(i -> notEmpty(arrayElement(splitByChar('|', host_proof), i)), [1,2,4,5,6]) " - "AND match(arrayElement(splitByChar('|', host_proof), 5), '^[1-9][0-9]*$') " - "AND match(arrayElement(splitByChar('|', host_proof), 6), '^[0-9]+$') " + f"SELECT participant_id, state, version, host_proof FROM {TABLE} FINAL " + f"WHERE {_CERTIFIED_RETIRED} " "AND heartbeat_at < now64(3) - toIntervalHour({hours:UInt32}) LIMIT 1000" ) _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 3c2309cc47..5872922964 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -939,6 +939,7 @@ "namespace_taxonomy_ttl_s": 300.0, "namespace_tune_document_sample": 50, "namespace_write_marks_flush_interval_s": 5.0, + "native_command_drain_timeout_s": 15.0, "oauth_access_token_ttl_seconds": 3600, "oauth_enabled": false, "oauth_issuer": "", From 17f8d534fdd0cebf2977a7e447b10c9aac0b045f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 20:55:54 +0000 Subject: [PATCH 114/181] Compare declaration work after parser initialization Signed-off-by: zlareb1 --- tests/unit/test_code_definition_family.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_code_definition_family.py b/tests/unit/test_code_definition_family.py index 5db75f4efb..1ee5950224 100644 --- a/tests/unit/test_code_definition_family.py +++ b/tests/unit/test_code_definition_family.py @@ -223,6 +223,9 @@ def work(n: int) -> int: def test_reading_many_declarators_of_one_binding_is_linear() -> None: from tests.unit.step_count import steps + # Load grammars and start the parser child before comparing per-file work. + assert file_definitions("warm.js", b"const warm = () => 0;\n") + def work(n: int) -> int: src = "const " + ",\n ".join(f"v{i} = () => {i} // c" for i in range(n)) + ";\n" return steps(lambda: file_definitions("a.js", src.encode())) From 295ea3ca2d11459b1526fe4eaaccca1bb582e282 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:00:59 +0000 Subject: [PATCH 115/181] Personal bearer subjects cannot be impersonated without a bot A service-account row cannot prove a token ID, session or parent credential remains live. Keep actual bearer ancestry checks, retain backend error identity, and align acted-for and doctor fixtures with their authorization contracts. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 8 ++-- docs/architecture/security.md | 14 +++--- server/auth/authz.py | 35 ++------------- tests/unit/test_cell_attach.py | 4 +- tests/unit/test_scim_admission.py | 74 +++++++++++++++++++++++++++++++ tests/unit/test_trusted_caller.py | 14 ++++-- 6 files changed, 104 insertions(+), 45 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 064e14b301..951b15560b 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -294,11 +294,13 @@ ancestor. Exemptions are closed: an active registered bot and its agents, including machine or workload exchanges backed by that registration, and a signed end user whose issuer and end-user metadata were verified. A prefix, an admin role or the absence of email -creates no exemption. Other `sa:` subjects use the current personal service-account -row: it must exist, remain unrevoked and unexpired, and have a creator. Its creator +creates no exemption. An actual bearer naming another `sa:` subject uses the current +personal service-account row: it must exist, remain unrevoked and unexpired, and have a creator. Its creator and every persisted lineage subject must qualify, even when grant inheritance is disabled. This security read bypasses the grant-inheritance metadata cache. Cycles -refuse; a shared ancestor completed on another path is checked once. +refuse; a shared ancestor completed on another path is checked once. A trusted caller +acts for an `sa:` subject only when it names an active registered bot or its agent; +a personal row cannot prove the token's, session's or parent credentials' liveness. One decision allows at most 1,024 supplied roots, distinct subjects, examined lineage edges, account rows and distinct emails in each dimension. Repeated root entries diff --git a/docs/architecture/security.md b/docs/architecture/security.md index a19e8e8b1f..b4f289f7f7 100644 --- a/docs/architecture/security.md +++ b/docs/architecture/security.md @@ -483,12 +483,14 @@ HTTP to act as another principal. The header's subject becomes the effective sub or deprovisioned, in any case and under any spelling it holds in the target org or the caller's own org (itself, and the handle and email of each active account of it whose email is verified), is not acted for (403), nor is a bot that is revoked outright or disabled, nor -any of its agents (a disable leaves their grants). A personal service-account subject -requires its current row to exist, be unrevoked and unexpired, and name a creator. In an -enforced cell with SCIM on, that creator and every lineage subject must also pass the -complete provisioning check; the inheritance cache supplies no security answer. A token -minted under a subject without `sa:` cannot be told from a person's, and is acted for until its subject is revoked -outright. A request that names no org, from a caller whose token is for every org, has no +any of its agents (a disable leaves their grants). Of the `sa:` subjects, only an active bot +and its agents are acted for: any other is a token's own subject, whose direct grants belong +to the token and must end with it, and a token ends in ways its directory row does not record +(an expiry, a revoke by `jti`, a cutoff on its maker or on itself). A trusted caller acting +for one would keep reaching what a dead token held. An actual personal bearer still needs +its current row, creator and every lineage subject to pass the complete provisioning check +in an enforced cell with SCIM on. A token minted under a subject without `sa:` cannot be +told from a person's, and is acted for until its subject is revoked outright. A request that names no org, from a caller whose token is for every org, has no org to read spellings in, so only the subject as sent is checked. An unverified email is not held: revoking a stranger's account that typed someone's email does not refuse acting for that email. Sign-up refuses a trusted caller's own subject as a name (the sign-up rule under diff --git a/server/auth/authz.py b/server/auth/authz.py index 59a40c95ca..d998e4a40b 100644 --- a/server/auth/authz.py +++ b/server/auth/authz.py @@ -86,7 +86,7 @@ class ActedForRefused(PermissionError): async def _check_provisioning(auth: AuthInfo | None, org: str | None) -> None: from server.auth import login, scim - from server.core.errors import BackendUnavailableError + from server.core.errors import BackendUnavailableError, LoomServerError if auth is None: return @@ -96,6 +96,8 @@ async def _check_provisioning(auth: AuthInfo | None, org: str | None) -> None: try: refused = await scim.refuses_auth(auth, org=scope_org) except login.SignInUnavailable as e: + if isinstance(e.__cause__, LoomServerError): + raise e.__cause__ from None raise BackendUnavailableError("SCIM provisioning cannot be checked yet") from e if refused: raise ActedForRefused(_ACTED_FOR_REVOKED) @@ -131,36 +133,7 @@ async def check_acted_for(auth: AuthInfo | None, org: str | None) -> None: for email in {s.lower() for s in spellings if "@" in s}: if await _remembered(("scim", email), _scim_deactivated): raise ActedForRefused(_ACTED_FOR_REVOKED) - from server.auth import scim - - if bot and not scim.requires_provisioning() and await _remembered(("bot", bot), _bot_ended): - await _check_personal_acted_for(org or auth.org, auth.subject, bot) - - -async def _check_personal_acted_for(org: str, subject: str, bot: str) -> None: - from server.auth import service_accounts - from server.core.ch import get_client, own_write_reads - from server.core.datetimes import as_aware_utc - from server.core.errors import BackendUnavailableError - - try: - with own_write_reads(): - machine = await get_client().query_one( - "SELECT active FROM loom.machine_identities FINAL " - "WHERE name = {name:String} LIMIT 1", - params={"name": bot}, - ) - row = None if machine is not None else await service_accounts._creator_row(org, subject) - except Exception as e: # unread status grants no acted-for authority - raise BackendUnavailableError("cannot read the acted-for service account") from e - if machine is not None: - if machine["active"]: - return - raise ActedForRefused(_ACTED_FOR_NO_BOT) - if row is None or not row["created_by"] or row["revoked_at"] is not None: - raise ActedForRefused(_ACTED_FOR_NO_BOT) - expiry = row.get("expires_at") - if expiry is not None and as_aware_utc(expiry).timestamp() <= time.time(): + if bot and await _remembered(("bot", bot), _bot_ended): raise ActedForRefused(_ACTED_FOR_NO_BOT) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 819f03d90f..d307b28a2e 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -1055,8 +1055,8 @@ async def test_doctor_names_both_catalog_checks_when_the_catalog_is_unread( async def before(_ch: Any) -> str: return "" - async def marked() -> tuple[str, bool, list[Any]]: - return "enforced", True, [] + async def marked() -> tuple[str, tuple[bool, str], bool, list[Any]]: + return "enforced", (True, "first"), True, [] async def unmarked() -> bool: return False diff --git a/tests/unit/test_scim_admission.py b/tests/unit/test_scim_admission.py index 497fe038db..0e6252c424 100644 --- a/tests/unit/test_scim_admission.py +++ b/tests/unit/test_scim_admission.py @@ -114,6 +114,80 @@ async def test_a_real_personal_service_account_uses_its_creator(people, token): ).subject == "sa:personal:one" +@pytest.mark.parametrize("required", [False, True]) +async def test_revoked_personal_credential_cannot_be_replaced_by_impersonation( + people, monkeypatch, required +): + from server.auth import gateway + from server.core import grant_cache + + subject = "sa:personal:one" + people.provision("alice@example.test") + people.services[subject] = {"created_by": "alice@example.test", "lineage": []} + people.bots["gateway"] = True + if not required: + monkeypatch.setattr(settings, "scim_token", "") + monkeypatch.setattr(settings, "trusted_callers", "sa:gateway") + rows = [] + identity_read = people.query_one + + async def query_one(sql, params=None): + if "loom.revoked_tokens" in sql: + assert params == {"k0": credential.token_id} + return {"v": 1} + return await identity_read(sql, params) + + async def insert(table, written): + assert table == "loom.revoked_tokens" + rows.extend(written) + + async def grant_map(org, held): + assert org == "acme" + return grant_cache.GrantMap(ranks={}, held=(held,)) + + monkeypatch.setattr(people, "query_one", query_one) + monkeypatch.setattr(people, "insert", insert, raising=False) + monkeypatch.setattr(grant_cache, "grant_map", grant_map) + raw = jwt.mint_token( + subject, + "acme", + ["*"], + ["writer"], + secret=SECRET, + is_service_account=True, + creator="alice@example.test", + ttl_seconds=60, + ) + credential = await jwt.authenticate_token(raw, secret=SECRET) + await authz.check_acted_for(credential, "acme") + live_row = dict(people.services[subject]) + await revocation.revoke(credential.token_id, reason="end the personal credential") + assert [row["jti"] for row in rows] == [credential.token_id] + assert revocation.is_revoked(credential.token_id) + assert not revocation.is_revoked(subject) + with pytest.raises(jwt.TokenError, match="revoked"): + jwt.verify_token(raw, secret=SECRET) + assert people.services[subject] == live_row + assert live_row.get("revoked_at") is live_row.get("expires_at") is None + + gateway_raw = jwt.mint_token( + "sa:gateway", + "acme", + ["*"], + ["writer"], + secret=SECRET, + is_service_account=True, + ttl_seconds=60, + ) + caller = await jwt.authenticate_token(gateway_raw, secret=SECRET) + effective = gateway.effective_auth(caller, subject) + assert effective.subject == subject + assert effective.token_id == caller.token_id != credential.token_id + assert effective.impersonated_by == caller.subject + with pytest.raises(authz.ActedForRefused, match="active bot"): + await authz.check_acted_for(effective, "acme") + + async def test_admission_outage_is_retryable_auth_unavailability(people, token): people.unavailable = True with pytest.raises(jwt.AuthUnavailable): diff --git a/tests/unit/test_trusted_caller.py b/tests/unit/test_trusted_caller.py index 0e9c9f77bf..5303154051 100644 --- a/tests/unit/test_trusted_caller.py +++ b/tests/unit/test_trusted_caller.py @@ -164,7 +164,7 @@ def test_agent_event_carries_impersonated_by() -> None: # ── the subject a trusted caller acts for ───────────────────────────────── -def _acting(subject: str, org: str = "clickhouse") -> AuthInfo: +def _acting(subject: str, org: str = "acme") -> AuthInfo: from server.auth.jwt import AuthInfo, Role return AuthInfo( @@ -203,7 +203,7 @@ async def grant_map(org_: str, subject: str) -> grant_cache.GrantMap: monkeypatch.setattr(grant_cache, "grant_map", grant_map) monkeypatch.setattr(revocation, "_revoked_folded", frozenset({revoked.lower()})) with pytest.raises(PermissionError, match="acts for is revoked"): - asyncio.run(authz.check_acted_for(_acting(header, org), "acme")) + asyncio.run(authz.check_acted_for(_acting(header, org), "acme" if org == "*" else org)) def test_an_unreadable_grant_map_refuses(monkeypatch: pytest.MonkeyPatch) -> None: @@ -298,8 +298,11 @@ async def grant_map(org: str, subject: str) -> grant_cache.GrantMap: def test_the_tokens_own_org_is_checked_too(monkeypatch: pytest.MonkeyPatch) -> None: """A personal namespace is provisioned in the token's org: a spelling tied to a revoked subject there refuses, though the target org ties nothing.""" + from dataclasses import replace + from server.auth import authz, revocation from server.core import grant_cache + from server.operations.base import Role async def grant_map(org: str, subject: str) -> grant_cache.GrantMap: tied = {"bob@corp.com", "bob"} if org == "clickhouse" else {subject} @@ -309,7 +312,12 @@ async def grant_map(org: str, subject: str) -> grant_cache.GrantMap: monkeypatch.setattr(revocation, "_revoked_folded", frozenset({"bob"})) # its own refusal, so the HTTP 403 adds no code-index reason to it with pytest.raises(authz.ActedForRefused, match="acts for is revoked"): - asyncio.run(authz.check_acted_for(_acting("bob@corp.com", "clickhouse"), "acme")) + asyncio.run( + authz.check_acted_for( + replace(_acting("bob@corp.com", "clickhouse"), roles=(Role.PLATFORM_ADMIN,)), + "acme", + ) + ) def test_an_end_user_token_is_not_checked_as_acted_for(monkeypatch: pytest.MonkeyPatch) -> None: From 925479048218828eac3eb9c9bd96ad8f29d73eda Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:05:11 +0000 Subject: [PATCH 116/181] Prove worker crash recovery within the fixture process namespace Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 153 ++++++++++++++++++ scripts/gates/docker_provider_smoke.py | 27 +++- tests/unit/test_docker_provider_smoke.py | 195 +++++++++++++++++++++++ 3 files changed, 370 insertions(+), 5 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 63b6097679..3b7265de77 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -1,12 +1,157 @@ """Seed/check nonempty rebuild surfaces inside the disposable Docker gate.""" import asyncio +import json +import os +import signal import sys +import time +from pathlib import Path +from server.core import provider_process from server.core.ch import get_client from server.insights.app.provider_command import model_command from server.settings import settings +SUPERVISOR = [ + "-m", + "uvicorn", + "docker_provider_app:app", + "--host", + "0.0.0.0", + "--port", + "7777", + "--workers", + "2", + "--timeout-graceful-shutdown", + "10", +] + + +def _process(pid: int) -> dict: + root = Path(f"/proc/{pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + return { + "pid": pid, + "parent": int(stat[1]), + "start": stat[19], + "command": (root / "cmdline").read_bytes().decode().rstrip("\0").split("\0"), + "namespace": os.stat(root / "ns/pid").st_ino, + "executable": os.readlink(root / "exe"), + } + + +def _supervisor() -> dict: + parent, me = _process(1), _process(os.getpid()) + assert parent["command"][1:] == SUPERVISOR, "Unexpected fixture supervisor" + assert parent["executable"] == me["executable"] + assert parent["namespace"] == me["namespace"] + return parent + + +def _worker_process(raw: str, parent: dict) -> dict: + proof = raw.split("|") + assert len(proof) == 9 and provider_process.has_certificate(raw) + assert proof[:4] == provider_process.host_proof().split("|")[:4] + process = _process(int(proof[4])) + assert process["pid"] > 1 + assert process["parent"] == 1 and process["start"] == proof[5] + assert process["namespace"] == parent["namespace"] + assert process["executable"] == parent["executable"] + command = process["command"] + assert len(command) == 4 and command[1] == "-c" + assert command[2].startswith("from multiprocessing.spawn import spawn_main; spawn_main(") + assert command[3] == "--multiprocessing-fork" + return process + + +def _workers(rows: list[dict], parent: dict) -> list[dict]: + here = provider_process.host_proof().split("|") + workers = [] + for row in rows: + proof = str(row["host_proof"]).split("|") + if proof[:4] != here[:4] or row["state"] == "retired": + continue + _worker_process(row["host_proof"], parent) + workers.append({key: row[key] for key in ("participant_id", "host_proof", "version")}) + assert len(workers) <= 2 + assert len({row["participant_id"] for row in workers}) == len(workers) + assert len({row["host_proof"].split("|")[4] for row in workers}) == len(workers) + return sorted(workers, key=lambda row: row["host_proof"]) + + +async def _worker_rows(client) -> list[dict]: + return await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE state != 'retired'" + ) + + +async def workers(client) -> list[dict]: + parent = _supervisor() + result = _workers(await _worker_rows(client), parent) + assert len(result) == 2 + assert _supervisor() == parent + return result + + +def _kill_worker(worker: dict, parent: dict) -> None: + proof = worker["host_proof"].split("|") + pid = int(proof[4]) + assert pid > 1 + captured = _worker_process(worker["host_proof"], parent) + fd = os.pidfd_open(pid) + try: + # A reused PID or changed supervisor refuses before the process-bound signal. + assert _process(pid) == captured and _supervisor() == parent + signal.pidfd_send_signal(fd, signal.SIGKILL) + finally: + os.close(fd) + + +async def retired(client, captured: list[dict]) -> bool: + from server.insights.store.provider_participants import running_queries + + rows = await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + if len(rows) != len(captured): + return False + expected = {row["participant_id"]: row for row in captured} + for row in rows: + old = expected[row["participant_id"]] + if ( + row["state"] != "retired" + or int(row["version"]) <= int(old["version"]) + or row["host_proof"] != old["host_proof"] + or not provider_process.has_certificate(row["host_proof"]) + or await running_queries(row["participant_id"], ch=client) != 0 + ): + return False + return True + + +async def crash_worker(client) -> None: + parent = _supervisor() + before = await workers(client) + victim = before[0] + _kill_worker(victim, parent) + deadline = time.monotonic() + 240 + while time.monotonic() < deadline: + assert _supervisor() == parent + if provider_process.proves_stopped(victim["host_proof"]) and await retired( + client, [victim] + ): + after = _workers(await _worker_rows(client), parent) + old = {row["participant_id"] for row in before} + new = {row["participant_id"] for row in after} + if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: + return + await asyncio.sleep(1) + raise AssertionError("Crashed worker retirement or supervised replacement was not proved") + async def seed() -> None: from server import boot @@ -75,6 +220,14 @@ async def main() -> None: try: if sys.argv[1] == "seed": await seed() + elif sys.argv[1] == "workers": + print(json.dumps(await workers(client))) + elif sys.argv[1] == "crash-worker": + await crash_worker(client) + elif sys.argv[1] == "retired": + assert await retired(client, json.loads(sys.argv[2])), ( + "Recreated workers did not retire" + ) else: await verify(sys.argv[1], int(sys.argv[2])) finally: diff --git a/scripts/gates/docker_provider_smoke.py b/scripts/gates/docker_provider_smoke.py index a71ec4cf9f..b5445e145e 100644 --- a/scripts/gates/docker_provider_smoke.py +++ b/scripts/gates/docker_provider_smoke.py @@ -59,9 +59,19 @@ def control(compose: list[str], failed_model: str) -> None: run(*compose, "exec", "-T", "provider-smoke", "python", "-c", script) -def data_check(compose: list[str], *args: str) -> None: +def data_check(compose: list[str], *args: str) -> list[dict] | None: source = (ROOT / "scripts/gates/docker_provider_data.py").read_text() + if args[0] == "workers": + result = subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + capture_output=True, + ) + return json.loads(result.stdout) run(*compose, "exec", "-T", "loom", "python", "-", *args, input=source, text=True) + return None def dashboard_action(base: str, action: str, *args: str) -> dict: @@ -105,6 +115,7 @@ def wait_admission(base: str) -> None: def smoke_providers(base: str, compose: list[str]) -> None: override = Path(compose[-1]) configuration = json.loads(override.read_text()) + # Match serve.sh's finite request grace so the lifespan fits Compose's stop grace. configuration["services"]["loom"]["entrypoint"] = [ "python", "-m", @@ -116,6 +127,8 @@ def smoke_providers(base: str, compose: list[str]) -> None: "7777", "--workers", "2", + "--timeout-graceful-shutdown", + "10", ] configuration["services"]["loom"]["volumes"] = [ f"{ROOT / 'tests/fixtures/docker_provider_app.py'}:/app/docker_provider_app.py:ro" @@ -171,15 +184,19 @@ def smoke_providers(base: str, compose: list[str]) -> None: control(compose, "") dashboard_action(base, "retry") - # SIGKILL exercises persistent process proofs and ambiguous-effect recovery, - # not just an orderly shutdown. Restart only this generated stack's Loom. - run(*compose, "kill", "-s", "SIGKILL", "loom") - run(*compose, "up", "-d", "--wait", "--wait-timeout", "180", "loom") + # Keep the supervisor's PID namespace: a different namespace cannot certify + # death. A live worker retires the exact crashed worker after its queries end. + data_check(compose, "crash-worker") status = wait_status(base, lambda row: row["pending"] is None) assert status["embedding_model"] == "docker-smoke-16" and status["embedding_dimensions"] == 16 wait_admission(base) data_check(compose, "docker-smoke-16", "16") + workers = data_check(compose, "workers") + assert workers is not None and len(workers) == 2 run(*compose, "up", "-d", "--force-recreate", "--wait", "--wait-timeout", "180", "loom") + # Recreation is graceful; prove the captured workers actually retired instead + # of treating Docker's container replacement as process-death evidence. + data_check(compose, "retired", json.dumps(workers)) status = request(base, "/v1/admin/models") assert status["source"] == "dashboard" and status["embedding_model"] == "docker-smoke-16" wait_admission(base) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index a561016e08..7e51d06878 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -1,5 +1,6 @@ """The smoke fixture is deterministic and isolates recovery from paid providers.""" +import io import json import threading import urllib.error @@ -138,3 +139,197 @@ async def test_data_check_requires_no_retained_rebuild_copies(monkeypatch): monkeypatch.setattr(data, "get_client", lambda: client) with pytest.raises(AssertionError): await data.verify("model", 8) + + +def test_crash_keeps_the_supervisor_and_verifies_retirement_before_reset(monkeypatch, tmp_path): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + compose = ["docker", "compose", "-f", str(override)] + events = [] + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + {**environment, "pending": None}, + ] + ) + model_reads = iter([environment, active]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + applications = iter(["queued", "active", "queued"]) + + def dashboard(_base, action, *_args): + events.append(action) + return {"state": next(applications) if action == "apply" else "queued"} + + def check(_compose, *args): + events.append(args[0]) + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + value = next(statuses) + assert predicate(value) + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *args: events.append(tuple(args))) + smoke.smoke_providers("http://fixture", compose) + assert not any(isinstance(event, tuple) and "SIGKILL" in event for event in events) + assert events.index("retry") < events.index("crash-worker") < events.index("workers") + recreate = [ + i + for i, event in enumerate(events) + if isinstance(event, tuple) and "--force-recreate" in event + ] + assert len(recreate) == 2 + assert events.index("workers") < recreate[-1] < events.index("retired") < events.index("reset") + + +def _process_fixture(monkeypatch): + parent = { + "pid": 1, + "parent": 0, + "start": "10", + "namespace": 1, + "executable": "/python", + "command": ["python", *data.SUPERVISOR], + } + processes = { + pid: { + "pid": pid, + "parent": 1, + "start": str(pid * 10), + "namespace": 1, + "executable": "/python", + "command": [ + "python", + "-c", + "from multiprocessing.spawn import spawn_main; spawn_main(tracker_fd=4, pipe_handle=8)", + "--multiprocessing-fork", + ], + } + for pid in (8, 9) + } + rows = [ + { + "participant_id": str(pid), + "version": 5, + "state": "running", + "host_proof": f"h|b|c|1|{pid}|{pid * 10}|||cell-retirement-v1", + } + for pid in (8, 9) + ] + monkeypatch.setattr(data.provider_process, "host_proof", lambda: "h|b|c|1|99|990||") + monkeypatch.setattr(data, "_process", lambda pid: processes[pid]) + return parent, processes, rows + + +def test_worker_selection_requires_registered_supervised_processes(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + foreign = {**rows[0], "host_proof": "h|b|c|2|8|80|||cell-retirement-v1"} + assert data._workers([*rows, foreign], parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + processes[8] = {**processes[8], "parent": 2} + with pytest.raises(AssertionError): + data._workers(rows, parent) + processes[8] = { + **processes[8], + "parent": 1, + "command": ["python", "-c", "from multiprocessing.resource_tracker import main; main(4)"], + } + with pytest.raises(AssertionError): + data._workers(rows, parent) + + +@pytest.mark.parametrize("fault", ["none", "pid-reused", "supervisor-changed"]) +def test_pidfd_crash_refuses_changed_identity_before_signalling(monkeypatch, fault): + parent, processes, rows = _process_fixture(monkeypatch) + events = [] + observed = iter( + [processes[8], {**processes[8], "start": "81"} if fault == "pid-reused" else processes[8]] + ) + monkeypatch.setattr(data, "_process", lambda _pid: next(observed)) + monkeypatch.setattr( + data, + "_supervisor", + lambda: {**parent, "start": "11"} if fault == "supervisor-changed" else parent, + ) + monkeypatch.setattr(data.os, "pidfd_open", lambda pid: events.append(("open", pid)) or 73) + monkeypatch.setattr(data.os, "close", lambda fd: events.append(("close", fd))) + monkeypatch.setattr( + data.signal, "pidfd_send_signal", lambda fd, sig: events.append(("signal", fd, sig)) + ) + if fault == "none": + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("signal", 73, data.signal.SIGKILL), ("close", 73)] + else: + with pytest.raises(AssertionError): + data._kill_worker(rows[0], parent) + assert events == [("open", 8), ("close", 73)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["none", "drained", "old-version", "different-proof", "busy", "unknown"] +) +async def test_retirement_requires_exact_newer_certificate_and_known_zero_queries( + monkeypatch, fault +): + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + _parent, _processes, rows = _process_fixture(monkeypatch) + observed = {**rows[0], "state": "retired", "version": 6} + if fault == "drained": + observed["state"] = "drained" + elif fault == "old-version": + observed["version"] = 5 + elif fault == "different-proof": + observed["host_proof"] = rows[1]["host_proof"] + client = AsyncMock() + client.query_all.return_value = [observed] + count = 1 if fault == "busy" else None if fault == "unknown" else 0 + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=count)) + assert await data.retired(client, [rows[0]]) is (fault == "none") + assert client.query_all.await_args.args[1] == {"ids": [rows[0]["participant_id"]]} From 74f1d6b1a1f518349590592920cb35f8d5cbd579 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:11:35 +0000 Subject: [PATCH 117/181] Workers enroll before their cell acceptance read Install the lifecycle role and enroll before caching primary acceptance, while keeping SCIM validation ahead of business services. Derive required provisioning from parsed SCIM credentials so blank entries retain the disabled policy. Signed-off-by: zlareb1 --- server/auth/scim_people.py | 3 +- server/worker_main.py | 13 ++++-- tests/unit/test_cell.py | 1 + tests/unit/test_scim_cell_lifecycle.py | 60 ++++++++++++++++++++++++++ tests/unit/test_scim_people.py | 9 ++++ tests/unit/test_workers_child.py | 2 + 6 files changed, 84 insertions(+), 4 deletions(-) diff --git a/server/auth/scim_people.py b/server/auth/scim_people.py index 033b198c49..fd1ba7302f 100644 --- a/server/auth/scim_people.py +++ b/server/auth/scim_people.py @@ -12,6 +12,7 @@ from server.core import cell, ch from server.core.datetimes import as_aware_utc from server.core.grant_sql import VERIFIED_EMAIL_SQL +from server.core.scim_tokens import tokens from server.settings import settings MAX_ITEMS = 1024 # not a setting: security work per complete decision @@ -58,7 +59,7 @@ def _decision_seconds() -> float: def requires_provisioning() -> bool: - return bool(settings.cell and settings.scim_token and cell.mode() == "enforced") + return bool(settings.cell and tokens(settings.scim_token) and cell.mode() == "enforced") async def refuses_subject( diff --git a/server/worker_main.py b/server/worker_main.py index f0aeabf40c..e11d244e27 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -28,12 +28,20 @@ log = logging.getLogger("loom.workers.child") +async def _enroll() -> None: + from server.core.ch import query_tag + from server.insights.app.provider_barrier import register_process + + with query_tag("boot.workers_child"): + retirement.get_gate().install("worker") + await register_process() + + async def _start_services(ch: Any) -> list[asyncio.Task[None]]: from server.code.app.index import repo_registry from server.core.background import register_buffer_drainer from server.core.ch import query_tag, set_query_tag from server.core.model_work_gate import get_model_work_gate - from server.insights.app.provider_barrier import register_process from server.insights.app.provider_runtime import load_active_configuration, refresh_loop from server.observability import get_emitter from server.tracker.app import registry_sources @@ -47,8 +55,6 @@ async def _trackers() -> None: # `with`, not set_query_tag: the supervisor and the shutdown run in this context after. with query_tag("boot.workers_child"): register_buffer_drainer("boot-writes", boot.drain_writes) - retirement.get_gate().install("worker") - await register_process() await load_active_configuration() await get_emitter().start() await registry_sources.refresh_db_sources(force=True, ch=ch) @@ -108,6 +114,7 @@ async def _serve(fd: int, alive_fd: int) -> None: await boot.open_storage_authority() services = [] try: + await _enroll() if settings.cell: # The passes verify tokens and judge the gate as the parent does: the child # reads the cell's rows itself. diff --git a/tests/unit/test_cell.py b/tests/unit/test_cell.py index 49220914c7..71ba8309ee 100644 --- a/tests/unit/test_cell.py +++ b/tests/unit/test_cell.py @@ -575,6 +575,7 @@ async def _beat(_fd) -> None: monkeypatch.setattr(worker_main.child, "beat", _beat) monkeypatch.setattr(worker_main, "get_supervisor", lambda: _Supervisor()) monkeypatch.setattr(worker_main, "_start_services", _services) + monkeypatch.setattr(worker_main, "_enroll", AsyncMock()) monkeypatch.setattr(boot, "start_cell_state", _cell_state) monkeypatch.setattr(boot, "open_storage_authority", _nothing) monkeypatch.setattr(boot, "close_storage_authority", _nothing) diff --git a/tests/unit/test_scim_cell_lifecycle.py b/tests/unit/test_scim_cell_lifecycle.py index 8ac922fd11..35f9b82d63 100644 --- a/tests/unit/test_scim_cell_lifecycle.py +++ b/tests/unit/test_scim_cell_lifecycle.py @@ -123,6 +123,7 @@ def test_worker_checks_cell_before_services_and_drains_before_leaving( ): from server import worker_main from server.core import ch + from server.insights.app import provider_barrier from server.workers import leader order = [] @@ -152,8 +153,67 @@ async def leave(*_args, **_kwargs): monkeypatch.setattr(boot, "close_storage_authority", AsyncMock()) monkeypatch.setattr(boot, "leave_deployment", leave) monkeypatch.setattr(boot, "start_cell_state", refuse) + monkeypatch.setattr(provider_barrier, "register_process", AsyncMock()) monkeypatch.setattr(worker_main, "_start_services", services) with pytest.raises(RuntimeError, match="SCIM token policy|services refused"): asyncio.run(worker_main._serve(0, 0)) assert order == (["cell"] if failure == "policy" else ["cell", "services"]) assert closed == ([] if failure == "policy" else [True]) + + +@pytest.mark.parametrize("raw,valid", [(OWN, True), (LEGACY, False)]) +async def test_worker_enrolls_before_acceptance_and_checks_policy_before_business( + lifecycle, monkeypatch, raw, valid +): + from server import worker_main + from server.core import ch, retirement + from server.insights.app import provider_barrier + from server.workers import leader + + events = [] + tasks = [] + observations = [] + gate = retirement.get_gate() + monkeypatch.setattr(settings, "scim_token", raw) + monkeypatch.setattr(provider_barrier, "_registered", False) + monkeypatch.setattr(provider_barrier, "_OWN", provider_barrier._Own()) + + class Heartbeat: + def start(self): + assert gate.role == "worker" + events.append("enroll") + + async def acceptance(*_a, **_kwargs): + observations.append((gate.role, gate.enrolled, provider_barrier._registered)) + events.append("acceptance") + return _row() + + def refresh_loop(): + tasks.append(asyncio.create_task(asyncio.Event().wait())) + return tasks[-1] + + async def business(_ch): + assert cell.gate_reason() == "" + events.append("business") + raise RuntimeError("business refused") + + async def leave(*_a, **_kwargs): + assert all(task.cancelled() for task in tasks) + + monkeypatch.setattr(provider_barrier, "_HEARTBEAT", Heartbeat()) + monkeypatch.setattr(pg, "fetchrow_reset", acceptance) + monkeypatch.setattr(cell, "start_refresh_loop", refresh_loop) + monkeypatch.setattr(worker_main.child, "read_deadlines", lambda *_a: None) + monkeypatch.setattr(worker_main.child, "exit_past_deadline", lambda *_a: None) + monkeypatch.setattr(worker_main.child, "beat", AsyncMock()) + monkeypatch.setattr(leader, "_CURRENT", leader.current()) + monkeypatch.setattr(ch, "get_client", lambda: type("Client", (), {"connect": AsyncMock()})()) + monkeypatch.setattr(boot, "open_storage_authority", AsyncMock()) + monkeypatch.setattr(boot, "close_storage_authority", AsyncMock()) + monkeypatch.setattr(boot, "leave_deployment", leave) + monkeypatch.setattr(worker_main, "_start_services", business) + with pytest.raises(RuntimeError, match="business refused|SCIM|SCIM_TOKEN"): + await worker_main._serve(0, 0) + assert observations == [("worker", True, True)] + assert events == (["enroll", "acceptance", "business"] if valid else ["enroll", "acceptance"]) + assert bool(tasks) is valid diff --git a/tests/unit/test_scim_people.py b/tests/unit/test_scim_people.py index 41afdf7357..d7e464cb0f 100644 --- a/tests/unit/test_scim_people.py +++ b/tests/unit/test_scim_people.py @@ -309,6 +309,15 @@ async def test_outage_is_retryable_and_never_cached(people): assert not await scim.refuses_subject("alice@example.test", org="acme") +@pytest.mark.parametrize("raw", ["", " ", " , ", ",,,"]) +async def test_empty_parsed_token_policy_keeps_retained_deactivations(people, monkeypatch, raw): + monkeypatch.setattr(settings, "scim_token", raw) + assert not scim.requires_provisioning() + people.provision("disabled@example.test", active=False) + assert not await scim.refuses_subject("missing@example.test", org="acme") + assert await scim.refuses_subject("disabled@example.test", org="acme") + + @pytest.mark.parametrize("scim_on", [False, True]) async def test_standalone_keeps_existing_deactivation_behavior(people, monkeypatch, scim_on): monkeypatch.setattr(settings, "cell", "") diff --git a/tests/unit/test_workers_child.py b/tests/unit/test_workers_child.py index 1f3c1854a6..e8e079a040 100644 --- a/tests/unit/test_workers_child.py +++ b/tests/unit/test_workers_child.py @@ -12,6 +12,7 @@ import threading import time from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -371,6 +372,7 @@ async def _retire() -> None: monkeypatch.setattr(ch_mod, "get_client", lambda: _Ch()) monkeypatch.setattr(worker_main, "_start_services", _start_services) + monkeypatch.setattr(provider_barrier, "register_process", AsyncMock()) monkeypatch.setattr(worker_main, "get_supervisor", lambda: _Sup()) monkeypatch.setattr(worker_main, "_drain", _drain) monkeypatch.setattr(provider_barrier, "retire_process", _retire) From a7aba18c644f9b87d5f833d9218379e82ac0ef27 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:19:29 +0000 Subject: [PATCH 118/181] Worker service loops finish before the final drain Cancel and await the service list on normal shutdown before stopping workers and draining writes, while retaining exceptional-startup cleanup. Signed-off-by: zlareb1 --- server/worker_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/worker_main.py b/server/worker_main.py index e11d244e27..029be53a03 100644 --- a/server/worker_main.py +++ b/server/worker_main.py @@ -126,6 +126,9 @@ async def _serve(fd: int, alive_fd: int) -> None: log.info("workers child %s stopping", os.getpid()) retirement.get_gate().close() embedder_calibration.stop_refresh_loop() + for task in services: + task.cancel() + await asyncio.gather(*services, return_exceptions=True) await supervisor.stop_all(drain_timeout_s=leader.drain_timeout_s()) await _drain() retirement.get_gate().stopped() From 724299bef7a674ccbeef8647e2b601c8c87b4bbe Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:30:27 +0000 Subject: [PATCH 119/181] Quickstart matches live workers to their registrations Select supervisor spawn children before joining participant history, and validate their exact inherited interpreter flags. Keep process identity and retirement checks before any signal. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 99 ++++++++++++++++++------ tests/unit/test_docker_provider_smoke.py | 77 +++++++++++++++++- 2 files changed, 148 insertions(+), 28 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 3b7265de77..726582244e 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -3,9 +3,11 @@ import asyncio import json import os +import re import signal import sys import time +from multiprocessing import spawn from pathlib import Path from server.core import provider_process @@ -44,39 +46,87 @@ def _process(pid: int) -> dict: def _supervisor() -> dict: parent, me = _process(1), _process(os.getpid()) assert parent["command"][1:] == SUPERVISOR, "Unexpected fixture supervisor" - assert parent["executable"] == me["executable"] - assert parent["namespace"] == me["namespace"] + assert parent["executable"] == me["executable"], "Fixture supervisor executable mismatch" + assert parent["namespace"] == me["namespace"], "Fixture supervisor PID namespace mismatch" return parent def _worker_process(raw: str, parent: dict) -> dict: proof = raw.split("|") - assert len(proof) == 9 and provider_process.has_certificate(raw) - assert proof[:4] == provider_process.host_proof().split("|")[:4] + assert len(proof) == 9 and provider_process.has_certificate(raw), "Worker certificate invalid" + assert proof[:4] == provider_process.host_proof().split("|")[:4], "Worker proof scope mismatch" process = _process(int(proof[4])) - assert process["pid"] > 1 - assert process["parent"] == 1 and process["start"] == proof[5] - assert process["namespace"] == parent["namespace"] - assert process["executable"] == parent["executable"] - command = process["command"] - assert len(command) == 4 and command[1] == "-c" - assert command[2].startswith("from multiprocessing.spawn import spawn_main; spawn_main(") - assert command[3] == "--multiprocessing-fork" + pid = process["pid"] + assert pid > 1, "Worker identity names the supervisor" + assert process["parent"] == 1, f"Worker PID {pid} parent mismatch: {process['parent']} != 1" + assert process["start"] == proof[5], f"Worker PID {pid} process-start proof mismatch" + assert process["namespace"] == parent["namespace"], f"Worker PID {pid} namespace mismatch" + assert process["executable"] == parent["executable"], f"Worker PID {pid} executable mismatch" + _spawn_command(process["command"], pid) return process +def _spawn_command(command: list[str], pid: int) -> None: + # The fixed supervisor and exec helper use the same executable and container + # startup flags. Let stdlib reproduce those flags, including inherited -B. + expected = spawn.get_command_line(tracker_fd=0, pipe_handle=0) + assert len(command) == len(expected) and command[1:-2] == expected[1:-2], ( + f"Worker PID {pid} interpreter flags or -c mismatch" + ) + assert command[-1] == "--multiprocessing-fork", f"Worker PID {pid} fork argument mismatch" + assert re.fullmatch( + r"from multiprocessing\.spawn import spawn_main; " + r"spawn_main\(tracker_fd=[0-9]+, pipe_handle=[0-9]+\)", + command[-2], + ), f"Worker PID {pid} spawn command mismatch" + + +def _kernel_workers() -> dict[int, dict]: + """The supervisor's actual spawn children, before consulting participant history.""" + found = {} + children = Path("/proc/1/task/1/children").read_text().split() + for value in children: + pid = int(value) + process = _process(pid) + command = process["command"] + if not any( + value.startswith("from multiprocessing.spawn import spawn_main;") for value in command + ): + continue # The multiprocessing resource tracker is not an application worker. + _spawn_command(command, pid) + assert process["parent"] == 1, f"Kernel child PID {pid} parent changed during census" + assert pid > 1 and pid not in found, "Invalid supervisor child PID census" + found[pid] = process + assert len(found) == 2, f"Expected two live supervisor spawn children; found {len(found)}" + return found + + def _workers(rows: list[dict], parent: dict) -> list[dict]: here = provider_process.host_proof().split("|") + children = _kernel_workers() workers = [] - for row in rows: - proof = str(row["host_proof"]).split("|") - if proof[:4] != here[:4] or row["state"] == "retired": - continue - _worker_process(row["host_proof"], parent) + for pid, process in children.items(): + matches = [] + for row in rows: + proof = str(row["host_proof"]).split("|") + if ( + len(proof) >= 6 + and row["state"] != "retired" + and proof[:4] == here[:4] + and proof[4] == str(pid) + and proof[5] == process["start"] + ): + matches.append(row) + assert len(matches) == 1, ( + f"Worker PID {pid} exact registrations: {len(matches)}; expected 1" + ) + row = matches[0] + assert _worker_process(row["host_proof"], parent) == process, ( + f"Worker PID {pid} changed during proof join" + ) workers.append({key: row[key] for key in ("participant_id", "host_proof", "version")}) - assert len(workers) <= 2 - assert len({row["participant_id"] for row in workers}) == len(workers) - assert len({row["host_proof"].split("|")[4] for row in workers}) == len(workers) + assert len({row["participant_id"] for row in workers}) == 2, "Worker participant IDs collide" + assert _kernel_workers() == children, "Supervisor worker identities changed during census" return sorted(workers, key=lambda row: row["host_proof"]) @@ -90,20 +140,21 @@ async def _worker_rows(client) -> list[dict]: async def workers(client) -> list[dict]: parent = _supervisor() result = _workers(await _worker_rows(client), parent) - assert len(result) == 2 - assert _supervisor() == parent + assert len(result) == 2, "Worker registration census is incomplete" + assert _supervisor() == parent, "Fixture supervisor changed during registration census" return result def _kill_worker(worker: dict, parent: dict) -> None: proof = worker["host_proof"].split("|") pid = int(proof[4]) - assert pid > 1 + assert pid > 1, "Refusing to signal the supervisor" captured = _worker_process(worker["host_proof"], parent) fd = os.pidfd_open(pid) try: # A reused PID or changed supervisor refuses before the process-bound signal. - assert _process(pid) == captured and _supervisor() == parent + assert _process(pid) == captured, f"Worker PID {pid} changed after pidfd open" + assert _supervisor() == parent, "Fixture supervisor changed before worker signal" signal.pidfd_send_signal(fd, signal.SIGKILL) finally: os.close(fd) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 7e51d06878..b1a472c394 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -225,6 +225,9 @@ def status(_base, predicate): def _process_fixture(monkeypatch): + from multiprocessing import spawn + from types import SimpleNamespace + parent = { "pid": 1, "parent": 0, @@ -241,10 +244,8 @@ def _process_fixture(monkeypatch): "namespace": 1, "executable": "/python", "command": [ - "python", - "-c", - "from multiprocessing.spawn import spawn_main; spawn_main(tracker_fd=4, pipe_handle=8)", - "--multiprocessing-fork", + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) ], } for pid in (8, 9) @@ -260,9 +261,77 @@ def _process_fixture(monkeypatch): ] monkeypatch.setattr(data.provider_process, "host_proof", lambda: "h|b|c|1|99|990||") monkeypatch.setattr(data, "_process", lambda pid: processes[pid]) + monkeypatch.setattr(data, "Path", lambda _path: SimpleNamespace(read_text=lambda: "8 9")) return parent, processes, rows +def test_worker_selection_joins_kernel_children_without_treating_other_participants_as_workers( + monkeypatch, +): + parent, processes, rows = _process_fixture(monkeypatch) + processes[44] = { + **processes[8], + "pid": 44, + "parent": 0, + "start": "440", + "command": ["python", "-", "seed"], + } + mixed = [ + { + **rows[0], + "participant_id": "native", + "host_proof": "h|b|c|1|44|440|||cell-retirement-v1", + }, + { + **rows[0], + "participant_id": "reused-pid", + "host_proof": "h|b|c|1|8|79|||cell-retirement-v1", + }, + {**rows[0], "participant_id": "gone", "host_proof": "h|b|c|1|66|660|||cell-retirement-v1"}, + *rows, + ] + assert data._workers(mixed, parent) == [ + {k: row[k] for k in ("participant_id", "host_proof", "version")} for row in rows + ] + assert [row["participant_id"] for row in mixed] == ["native", "reused-pid", "gone", "8", "9"] + + +def test_worker_command_uses_exact_inherited_interpreter_flags(monkeypatch): + from multiprocessing import spawn, util + + parent, processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(util, "_args_from_interpreter_flags", lambda: ["-B"]) + command = [ + str(value, "utf-8") if isinstance(value, bytes) else value + for value in spawn.get_command_line(tracker_fd=4, pipe_handle=8) + ] + processes[8] = {**processes[8], "command": command} + assert data._worker_process(rows[0]["host_proof"], parent) == processes[8] + for flags in (["-E"], ["-B", "-X", "unknown"]): + processes[8] = {**processes[8], "command": [command[0], *flags, *command[-3:]]} + with pytest.raises(AssertionError, match="interpreter flags"): + data._worker_process(rows[0]["host_proof"], parent) + + +def test_kernel_workers_never_ignore_missing_or_mismatched_current_proofs(monkeypatch): + parent, processes, rows = _process_fixture(monkeypatch) + for proof in ( + None, + "h|b|c|1|8|79|||cell-retirement-v1", + "h|b|c|2|8|80|||cell-retirement-v1", + "h|b|c|1|8|80|||unknown", + ): + changed = rows[1:] if proof is None else [{**rows[0], "host_proof": proof}, rows[1]] + with pytest.raises(AssertionError, match="exact registrations|certificate invalid"): + data._workers(changed, parent) + for field, value in (("namespace", 2), ("executable", "/other-python")): + original = processes[8] + processes[8] = {**original, field: value} + with pytest.raises(AssertionError, match=f"{field} mismatch"): + data._workers(rows, parent) + processes[8] = original + + def test_worker_selection_requires_registered_supervised_processes(monkeypatch): parent, processes, rows = _process_fixture(monkeypatch) foreign = {**rows[0], "host_proof": "h|b|c|2|8|80|||cell-retirement-v1"} From 824870fef02926c70129d15c2b44fd371705a49e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:37:39 +0000 Subject: [PATCH 120/181] Caller routes check orgless bearers in the configured signup org Preserve the two caller-route fallbacks through middleware and downstream authentication without changing the token identity or passing a placement principal. Keep concrete-org authorization and complete-person checks, and match verifier mock keyword signatures. Signed-off-by: zlareb1 --- server/auth/bearer.py | 18 +++++++--- server/auth/jwt.py | 17 +++++++-- server/identity/app/me_callers.py | 4 ++- server/transport/tenancy_gate.py | 15 ++++++-- tests/unit/test_cell_tenancy.py | 20 ++++++----- tests/unit/test_scim_admission.py | 58 +++++++++++++++++++++++++++++++ tests/unit/test_sharing.py | 2 +- 7 files changed, 114 insertions(+), 20 deletions(-) diff --git a/server/auth/bearer.py b/server/auth/bearer.py index 7a6a00deaf..0db57bbcda 100644 --- a/server/auth/bearer.py +++ b/server/auth/bearer.py @@ -50,7 +50,7 @@ def missing_bearer() -> HTTPException: async def verify_bearer_header( - authorization: str | None, *, acts_for_end_users: bool = False + authorization: str | None, *, acts_for_end_users: bool = False, fallback_org: str | None = None ) -> AuthInfo | None: """The principal an `Authorization` header names, None when it carries no credential; HTTPException 401 / 503 as `verify_bearer`, and 403 for an @@ -64,7 +64,13 @@ async def verify_bearer_header( status_code=401, detail={"code": "AUTH_INVALID", "message": str(e)}, ) from e - info = None if token is None else await verify_bearer(token) + info = None + if token is not None: + info = ( + await verify_bearer(token, fallback_org=fallback_org) + if fallback_org is not None + else await verify_bearer(token) + ) if info is not None and not acts_for_end_users and is_end_user(info): raise gateway_refused() return info @@ -93,12 +99,16 @@ def auth_unavailable(e: AuthUnavailable) -> HTTPException: ) -async def verify_bearer(token: str) -> AuthInfo: +async def verify_bearer(token: str, *, fallback_org: str | None = None) -> AuthInfo: """The principal `token` names; HTTPException 401 / 503 otherwise. The rest of the request's ClickHouse queries carry its quota key, so routes outside the op registry draw the caller's bucket too.""" try: - info = await authenticate_token(token) + info = ( + await authenticate_token(token, fallback_org=fallback_org) + if fallback_org is not None + else await authenticate_token(token) + ) except TokenError as e: record_auth_refusal(e, transport="http") if isinstance(e, AuthUnavailable): diff --git a/server/auth/jwt.py b/server/auth/jwt.py index 8eeab98ee6..52eb637519 100644 --- a/server/auth/jwt.py +++ b/server/auth/jwt.py @@ -509,13 +509,24 @@ def _end_user_key(token: str) -> tuple[str, EndUserIssuer, Any] | None: ) -async def authenticate_token(token: str, *, secret: str | None = None) -> AuthInfo: - """Verify the credential, then admit its complete person and ancestry graph.""" +async def authenticate_token( + token: str, *, secret: str | None = None, fallback_org: str | None = None +) -> AuthInfo: + """Verify the credential, then admit its complete person and ancestry graph. + + Caller routes supply their configured fallback for an org-less or wildcard bearer; + the returned token identity remains unchanged. + """ from server.auth import login, scim auth = verify_token(token, secret=secret) + org = fallback_org if auth.org in ("", "*") else None try: - refused = await scim.refuses_auth(auth) + refused = ( + await scim.refuses_auth(auth, org=org) + if org is not None + else await scim.refuses_auth(auth) + ) except login.SignInUnavailable as e: raise AuthUnavailable( "SCIM provisioning cannot be checked yet; retry shortly", reason="provisioning" diff --git a/server/identity/app/me_callers.py b/server/identity/app/me_callers.py index 11450e3985..95018d7ddb 100644 --- a/server/identity/app/me_callers.py +++ b/server/identity/app/me_callers.py @@ -138,7 +138,9 @@ async def caller_identity(request: Request) -> tuple[str, str]: """The supplied bearer's identity, or the deliberate no-bearer dev identity.""" from server.auth.bearer import missing_bearer, verify_bearer_header - info = await verify_bearer_header(request.headers.get("Authorization")) + info = await verify_bearer_header( + request.headers.get("Authorization"), fallback_org=settings.self_signup_org + ) if info is None: if settings.auth_actually_required(): raise missing_bearer() diff --git a/server/transport/tenancy_gate.py b/server/transport/tenancy_gate.py index d8c51a1c61..1c992f6006 100644 --- a/server/transport/tenancy_gate.py +++ b/server/transport/tenancy_gate.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +from dataclasses import replace from typing import TYPE_CHECKING, Any from starlette.responses import JSONResponse @@ -73,7 +74,14 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: try: token = _bearer(scope) - info = await authenticate_token(token) if token is not None else None + fallback = settings.self_signup_org if path in _SIGN_UP_ORG_ROUTES else None + info = None + if token is not None: + info = ( + await authenticate_token(token, fallback_org=fallback) + if fallback is not None + else await authenticate_token(token) + ) except AuthUnavailable as e: # Cannot judge the token yet: refused, never passed on unchecked. The caller # gets the error's kind; the text is logged. @@ -101,7 +109,10 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: org = acting_org(path, info) from server.auth.authz import check_acted_for - await check_acted_for(info, org) + caller = ( + replace(info, org=org) if not info.org and path in _SIGN_UP_ORG_ROUTES else info + ) + await check_acted_for(caller, org) # The sign-up org a route falls back to is not the token's: no principal, so # it is recorded or refused, never placed, as for a sign-up. await tenancy.ensure_held(org, info if org == info.org else None) diff --git a/tests/unit/test_cell_tenancy.py b/tests/unit/test_cell_tenancy.py index 6192b5a696..52dcd090f6 100644 --- a/tests/unit/test_cell_tenancy.py +++ b/tests/unit/test_cell_tenancy.py @@ -365,7 +365,7 @@ async def test_the_middleware_checks_a_raw_routes_token_org( from server.transport import tenancy_gate monkeypatch.setattr(settings, "cell", "eu-1") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("other")) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("other")) async def _held(org: Any, auth: Any) -> None: if org != "held": @@ -717,7 +717,7 @@ async def test_on_a_token_route_a_bearer_that_does_not_verify_passes_to_the_rout from server.auth import jwt as jwt_mod from server.transport import tenancy_gate - def _refuse(_token: str) -> AuthInfo: + def _refuse(_token: str, *, secret: str | None = None) -> AuthInfo: raise jwt_mod.TokenError("signature does not verify") monkeypatch.setattr(settings, "cell", "eu-1") @@ -872,7 +872,7 @@ async def test_the_middleware_answers_503_while_it_cannot_judge_the_token( monkeypatch.setattr(settings, "cell", "eu-1") - def _unavailable(_t: str) -> None: + def _unavailable(_t: str, *, secret: str | None = None) -> None: raise jwt.KeyringUnavailable("the signing keyring is not loaded yet") monkeypatch.setattr("server.auth.jwt.verify_token", _unavailable) @@ -1072,7 +1072,7 @@ async def test_an_org_wide_token_on_the_callers_own_routes_is_checked_in_its_fal monkeypatch.setattr(settings, "cell", "eu-1") monkeypatch.setattr(settings, "self_signup_org", "signups") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("*")) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("*")) checked: list[str] = [] async def _held(org: Any, auth: Any) -> None: @@ -1128,7 +1128,9 @@ async def test_on_a_scim_route_the_middleware_answers_an_rfc_7644_error( from server.transport import tenancy_gate monkeypatch.setattr(settings, "cell", "eu-1") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("beta", Role.ORG_ADMIN)) + monkeypatch.setattr( + "server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("beta", Role.ORG_ADMIN) + ) async def _wrong(org: Any, auth: Any) -> None: raise errors.WrongCellError(str(org), cell="us-1") @@ -1166,7 +1168,7 @@ async def test_an_org_less_or_org_wide_token_on_the_callers_routes_is_served_in_ monkeypatch.setattr(settings, "cell", "eu-1") monkeypatch.setattr(settings, "self_signup_org", "signups") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth(token_org)) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth(token_org)) seen: list[tuple[str, Any]] = [] async def _held(org: Any, auth: Any) -> None: @@ -1196,7 +1198,7 @@ async def test_an_org_wide_tokens_sign_up_org_is_recorded_or_refused_never_place cell_pg.mode = mode calls = _router(monkeypatch, {"cell": "eu-1", "deployment_id": "dep-eu-1"}) monkeypatch.setattr(settings, "self_signup_org", "signups") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("*")) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("*")) app, send = AsyncMock(), AsyncMock() scope = { "type": "http", @@ -1226,7 +1228,7 @@ async def test_the_service_account_routes_act_in_the_tokens_own_org( cell_pg.down = True # nothing is read for `*` monkeypatch.setattr(settings, "self_signup_org", "signups") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("*")) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("*")) for path in ("/v1/me/service_accounts", "/v1/me/service_accounts.create"): app = AsyncMock() scope = { @@ -1247,7 +1249,7 @@ async def test_the_middlewares_503_carries_the_errors_own_fields( from server.transport import tenancy_gate monkeypatch.setattr(settings, "cell", "eu-1") - monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t: _auth("acme")) + monkeypatch.setattr("server.auth.jwt.verify_token", lambda _t, *, secret=None: _auth("acme")) async def _down(org: Any, auth: Any) -> None: raise errors.BackendUnavailableError("router down", operation="placement") diff --git a/tests/unit/test_scim_admission.py b/tests/unit/test_scim_admission.py index 0e6252c424..578460d4d2 100644 --- a/tests/unit/test_scim_admission.py +++ b/tests/unit/test_scim_admission.py @@ -442,3 +442,61 @@ def name(node, aliases=imports, is_jwt=path.name == "jwt.py"): ("server/auth/jwt.py", "authenticate_token"), ("server/auth/oauth.py", "revoke_token"), } + + +@pytest.mark.parametrize("path", ["/v1/me/sharing-summary", "/v1/me/set-default-namespace"]) +@pytest.mark.parametrize("parent", ["", "missing@example.test"]) +async def test_caller_orgless_fallback_checks_the_complete_person_in_the_signup_org( + people, token, monkeypatch, path, parent +): + import httpx + + from server.auth import tenancy + from server.transport.tenancy_gate import TenancyMiddleware + + monkeypatch.setattr(settings, "self_signup_org", "signups") + people.accounts.append(("signups", "alice", "alice@example.test", True)) + people.provision("alice@example.test") + raw = jwt.mint_token( + "alice", + "", + ["*"], + ["writer"], + secret=SECRET, + ttl_seconds=60, + parent_subjects=[parent] if parent else [], + parent_iats=[int(time.time())] if parent else [], + ) + held = [] + admitted = [] + + async def ensure_held(org, principal): + held.append((org, principal)) + + monkeypatch.setattr(tenancy, "ensure_held", ensure_held) + app = FastAPI() + app.add_middleware(TenancyMiddleware) + + @app.get(path) + async def caller(request: Request): + subject, org = await me_callers.caller_identity(request) + admitted.append((subject, org)) + return {"subject": subject, "org": org} + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app), base_url="http://test" + ) as client: + response = await client.get( + path, params={"org": "foreign"}, headers={"Authorization": f"Bearer {raw}"} + ) + if parent: + assert response.status_code == 401 + assert response.json()["detail"]["code"] == "AUTH_INVALID" + assert admitted == held == [] + else: + assert response.status_code == 200 + assert admitted == [("alice", "signups")] + assert held == [("signups", None)] + original = await jwt.authenticate_token(raw, secret=SECRET, fallback_org="signups") + assert original.subject == "alice" and original.org == "" + assert {p["org"] for p in people.reads if "org" in p} == {"signups"} diff --git a/tests/unit/test_sharing.py b/tests/unit/test_sharing.py index 52fbb30581..2e23cc9b50 100644 --- a/tests/unit/test_sharing.py +++ b/tests/unit/test_sharing.py @@ -865,7 +865,7 @@ async def test_mcp_refuses_an_end_user_token_as_not_allowed_not_as_invalid( from server.auth import jwt from server.transport import mcp_auth - monkeypatch.setattr(jwt, "verify_token", lambda _token: _user("alice")) + monkeypatch.setattr(jwt, "verify_token", lambda _token, *, secret=None: _user("alice")) request = SimpleNamespace(headers={"authorization": "Bearer eu"}) with pytest.raises(PermissionError) as refused: (await mcp_auth._verify_bearer(request)) # type: ignore[arg-type] From 727e60166cf83e02c4ab76bc4d2de9216605a4a9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:46:16 +0000 Subject: [PATCH 121/181] Trigger acceptance runs on isolated CI stores Create a capped ClickHouse and synchronous Postgres pair for the fixed API and delivery experiment. Bind the actual checkout, interpreter and workload bytes, preserve retirement before owned store cleanup, and publish sanitized evidence. Signed-off-by: zlareb1 --- .github/workflows/docker-smoke.yml | 54 + scripts/gates/cell_trigger_acceptance.py | 883 ++++++ .../testsupport/cell_trigger/__init__.py | 1 + .../gates/testsupport/cell_trigger/capture.py | 1024 +++++++ .../gates/testsupport/cell_trigger/fixture.py | 1250 +++++++++ .../gates/testsupport/cell_trigger/native.py | 2391 +++++++++++++++++ .../cell_trigger/owned_children.py | 145 + .../testsupport/cell_trigger/protocol.json | 235 ++ .../testsupport/cell_trigger/tls_proxy.py | 168 ++ tests/unit/test_cell_trigger_ci_creator.py | 165 ++ 10 files changed, 6316 insertions(+) create mode 100644 scripts/gates/cell_trigger_acceptance.py create mode 100644 scripts/gates/testsupport/cell_trigger/__init__.py create mode 100644 scripts/gates/testsupport/cell_trigger/capture.py create mode 100644 scripts/gates/testsupport/cell_trigger/fixture.py create mode 100644 scripts/gates/testsupport/cell_trigger/native.py create mode 100644 scripts/gates/testsupport/cell_trigger/owned_children.py create mode 100644 scripts/gates/testsupport/cell_trigger/protocol.json create mode 100644 scripts/gates/testsupport/cell_trigger/tls_proxy.py create mode 100644 tests/unit/test_cell_trigger_ci_creator.py diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml index 3088015c40..05f3564698 100644 --- a/.github/workflows/docker-smoke.yml +++ b/.github/workflows/docker-smoke.yml @@ -12,6 +12,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -45,6 +49,10 @@ on: "server/**", "schemas/**", "scripts/gates/docker*", + "scripts/gates/cell_trigger_acceptance.py", + "scripts/gates/testsupport/cell_trigger/**", + "bench/cell_trigger_gate.py", + "tests/unit/test_cell_trigger_ci_creator.py", "frontend/**", "tests/fixtures/docker_provider_app.py", "scripts/install-code-toolchains.sh", @@ -255,3 +263,49 @@ jobs: -v "$RUNNER_TEMP/pylib:/pylib:ro" -e PYTHONPATH=/pylib -e PYTHONDONTWRITEBYTECODE=1 \ -e LOOM_ENV_FILE= -e LOOM_TEST_REQUIRE_OWNER_CHECK=1 --entrypoint python loom:sandbox-ci \ -m pytest -q -p no:cacheprovider tests/unit/test_scip_checkout_fetch_ref.py + + cell-trigger-acceptance: + name: Cell trigger end-to-end acceptance + if: github.event_name == 'pull_request' || github.event_name == 'push' + needs: clickhouse + runs-on: ubuntu-24.04-arm + timeout-minutes: 150 + env: + PYTHONDONTWRITEBYTECODE: "1" + LOOM_ENV_FILE: "" + LOOM_SETTINGS_FILE: "" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + id: python + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.12.19" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install the locked native dependencies + run: uv sync --locked --python "${{ steps.python.outputs.python-path }}" --extra dev --extra code + - name: Check native tools + run: | + command -v docker + command -v openssl + docker info >/dev/null + - name: Check triggers against real API writes and delivery passes + env: + CLICKHOUSE_IMAGE: ${{ needs.clickhouse.outputs.image }} + PYTHONPYCACHEPREFIX: ${{ runner.temp }}/trigger-bytecode + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --clickhouse-image "$CLICKHOUSE_IMAGE" + - name: Remove only this attempt's created stores + if: always() + run: .venv/bin/python -B scripts/gates/cell_trigger_acceptance.py --cleanup-only + - name: Retain sanitized trigger acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: cell-trigger-acceptance + path: ${{ runner.temp }}/cell-trigger-artifacts + retention-days: 7 diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py new file mode 100644 index 0000000000..97eedfeeab --- /dev/null +++ b/scripts/gates/cell_trigger_acceptance.py @@ -0,0 +1,883 @@ +"""Run the fixed trigger acceptance workload on fresh, job-owned stores.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import time +import uuid +from pathlib import Path + +GIB = 1024**3 +SUPPORT = Path(__file__).resolve().parent / "testsupport/cell_trigger" + + +def require(ok: object, message: str) -> None: + if not ok: + raise RuntimeError(message) + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def failure_site(error: BaseException | None) -> dict | None: + if error is None: + return None + site = {"file": Path(__file__).name, "line": 0, "stage": "unknown"} + trace = error.__traceback__ + while trace is not None: + if ( + Path(trace.tb_frame.f_code.co_filename).resolve() == Path(__file__).resolve() + and trace.tb_frame.f_code.co_name != "require" + ): + site.update(line=trace.tb_lineno, stage=trace.tb_frame.f_code.co_name[:64]) + trace = trace.tb_next + return site + + +def source_hashes(root: Path) -> dict: + paths = { + name: SUPPORT / name + for name in ( + "fixture.py", + "native.py", + "tls_proxy.py", + "capture.py", + "owned_children.py", + "protocol.json", + ) + } + paths.update( + driver=root / "bench/cell_trigger_gate.py", + entrypoint=Path(__file__).resolve(), + workflow=root / ".github/workflows/docker-smoke.yml", + ) + return {name: sha(path) for name, path in paths.items()} + + +def write(path: Path, value: object, *, replace: bool = False) -> None: + target = path.with_name(path.name + ".tmp") if replace else path + with target.open("x", encoding="utf-8") as stream: + os.chmod(target, 0o600) + json.dump(value, stream, indent=2) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + if replace: + os.replace(target, path) + + +def source(root: Path) -> dict: + def git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=root, text=True, timeout=10).strip() + + require(not git("status", "--porcelain"), "Acceptance requires a clean checkout") + return {"head": git("rev-parse", "HEAD"), "tree": git("rev-parse", "HEAD^{tree}")} + + +def resources(path: Path, available_gib: int = 10) -> None: + available = next( + int(line.split()[1]) + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + require(available >= available_gib * 1024**2, "Acceptance available RAM floor") + require(shutil.disk_usage(path).free >= 16 * GIB, "Acceptance requires 16 GiB free disk") + + +def ports(count: int) -> list[int]: + sockets = [] + try: + for _ in range(count): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sockets.append(sock) + return [sock.getsockname()[1] for sock in sockets] + finally: + for sock in sockets: + sock.close() + + +def container_matches(actual: dict, expected: dict, label: str) -> bool: + return ( + actual.get("Id") == expected["id"] + and actual.get("Created") == expected["created"] + and actual.get("Config", {}).get("Labels", {}).get("loom.ci.trigger") == label + ) + + +def native_cleanup_complete(folder: Path, fixture_id: str) -> None: + fixture = folder / "fixture" + if not fixture.exists(): + return # Native validation never began; only the creator's stores exist. + receipt = fixture / "outer-result.json" + require( + not receipt.is_symlink() + and receipt.stat().st_uid == os.geteuid() + and receipt.stat().st_size <= 65536, + "Private native cleanup receipt", + ) + value = json.loads(receipt.read_text()) + require( + value.get("schema") == 2 + and value.get("fixture_id") == fixture_id + and value.get("native_cleanup_complete") is True, + "Native retirement, scratch cleanup or child reap remains incomplete", + ) + + +class Stores: + def __init__(self, root: Path, folder: Path, fixture_id: str, run: dict) -> None: + self.root, self.folder, self.fixture_id, self.run = root, folder, fixture_id, run + self.items: list[dict] = [] + self.network = None + self.prefix = "loom-trigger-" + fixture_id[:12] + self.cancelled = 0 + self.terminal = False + self.creation_deadline = time.monotonic() + 900 + self.env = { + "PATH": os.environ["PATH"], + "HOME": str(folder), + "LANG": "C.UTF-8", + "DOCKER_CONFIG": str(folder / "docker"), + } + + def save(self) -> None: + write( + self.folder / "stores.json", + { + "fixture_id": self.fixture_id, + "ci_run": self.run, + "items": self.items, + "network": self.network, + }, + replace=True, + ) + + def interrupted(self, signum: int, _frame: object) -> None: + self.cancelled = self.cancelled or signum + if self.terminal: + raise SystemExit(128 + self.cancelled) + + def command(self, *argv: str, timeout: int = 120, cleanup: bool = False) -> str: + if not cleanup: + require(not self.cancelled, "Acceptance creation cancelled") + resources(self.folder, 4) + remaining = self.creation_deadline - time.monotonic() + require(remaining > 0, "Owned store creation deadline") + timeout = min(timeout, max(1, int(remaining))) + result = subprocess.run(argv, env=self.env, capture_output=True, text=True, timeout=timeout) + require(len(result.stdout) + len(result.stderr) <= 8 * 1024**2, "Creator output bound") + require(result.returncode == 0, "Owned store command failed: " + argv[0]) + return result.stdout.strip() + + def inspect(self, identity: str, *, cleanup: bool = False) -> dict: + rows = json.loads(self.command("docker", "inspect", identity, cleanup=cleanup)) + require(len(rows) == 1, "Exactly one owned container") + return rows[0] + + def create(self, role: str, args: list[str]) -> dict: + name = self.prefix + "-" + role + cidfile = self.folder / (role + ".cid") + # The intent precedes Docker. A failed create is resolved by its unique label. + intent = {"role": role, "name": name, "id": None, "created": None} + self.items.append(intent) + self.save() + self.command( + "docker", + "create", + "--name", + name, + "--cidfile", + str(cidfile), + "--label", + "loom.ci.trigger=" + self.fixture_id, + "--label", + "loom.ci.run=" + self.run["run_id"], + *args, + ) + identity = cidfile.read_text().strip() + require(re.fullmatch(r"[a-f0-9]{64}", identity), "Actual created container ID") + actual = self.inspect(identity) + require( + actual["Name"] == "/" + name + and actual["Config"]["Labels"]["loom.ci.trigger"] == self.fixture_id, + "Created container identity", + ) + intent.update(id=identity, created=actual["Created"]) + self.save() + write( + self.folder / (role + "-creator.json"), + { + "task": "cell-trigger-acceptance", + "ci_run": self.run, + "container_id": identity, + "created_at": actual["Created"], + }, + ) + resources(self.folder) + self.command("docker", "start", identity) + return intent + + def wait(self, probe, seconds: int = 120) -> None: + end = time.monotonic() + seconds + while True: + require(not self.cancelled, "Owned store readiness cancelled") + try: + if probe(): + return + except (RuntimeError, subprocess.SubprocessError, ValueError): + pass + require(time.monotonic() < end, "Owned store readiness deadline") + time.sleep(1) + + def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: + selected = ports(8) + ch_port, native_port, pg_port, standby_port = selected[:4] + self.command("docker", "pull", ch_image, timeout=300) + self.command("docker", "pull", "postgres:16", timeout=300) + network_name = self.prefix + "-net" + self.network = {"name": network_name, "id": None} + self.save() + self.network["id"] = self.command( + "docker", + "network", + "create", + "--label", + "loom.ci.trigger=" + self.fixture_id, + network_name, + ) + self.save() + memory_config = self.folder / "memory.xml" + memory_config.write_text( + "4831838208\n" + ) + memory_config.chmod(0o644) + ch_data = self.folder / "clickhouse-data" + ch_data.mkdir(mode=0o777) + ch_data.chmod(0o777) + mounts = [ + (ch_data, "/var/lib/clickhouse", False), + (self.root / "deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3", True), + (self.folder / "memory.xml", "/etc/clickhouse-server/config.d/memory.xml", True), + ] + for filename, section in ( + ("lemmatizers.xml", "config.d"), + ("nlp_user_settings.xml", "users.d"), + ("keeper.xml", "config.d"), + ("access_control.xml", "config.d"), + ): + mounts.append( + ( + self.root / "deploy/clickhouse-config" / filename, + "/etc/clickhouse-server/" + section + "/" + filename, + True, + ) + ) + for filename in ("log_queries_readonly.xml", "custom_settings_prefixes.xml"): + mounts.append( + ( + self.root / "dev/clickhouse-ci" / filename, + "/etc/clickhouse-server/users.d/" + filename, + True, + ) + ) + ch_args = [ + "--memory", + str(5 * GIB), + "--memory-swap", + str(5 * GIB), + "-p", + f"127.0.0.1:{ch_port}:8123", + "-p", + f"127.0.0.1:{native_port}:9000", + "-e", + "CLICKHOUSE_USER=loom", + "-e", + "CLICKHOUSE_PASSWORD=loomdev", + "-e", + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1", + "--ulimit", + "nofile=262144:262144", + ] + for host, target, readonly in mounts: + ch_args += ["-v", str(host) + ":" + target + (":ro" if readonly else "")] + ch = self.create("clickhouse", [*ch_args, ch_image]) + pg_data = self.folder / "pg-data" + pg_data.mkdir() + pg = self.create( + "pg-primary", + [ + "--network", + network_name, + "--network-alias", + "primary", + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{pg_port}:5432", + "-v", + str(pg_data) + ":/var/lib/postgresql/data", + "-e", + "POSTGRES_USER=loom", + "-e", + "POSTGRES_PASSWORD=loomdev", + "-e", + "POSTGRES_DB=loom", + "postgres:16", + "-c", + "wal_level=logical", + "-c", + "max_wal_senders=10", + "-c", + "max_replication_slots=10", + "-c", + "synchronous_commit=on", + "-c", + "shared_buffers=128MB", + ], + ) + psql = ["docker", "exec", pg["id"], "psql", "-U", "loom", "-d", "loom", "-Atc"] + self.wait(lambda: self.command(*psql, "SELECT 1") == "1") + self.command(*psql, "CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator'") + self.command( + "docker", + "exec", + pg["id"], + "bash", + "-c", + "echo 'host replication replicator all scram-sha-256' >> \"$PGDATA/pg_hba.conf\"", + ) + self.command(*psql, "ALTER SYSTEM SET synchronous_standby_names = 'FIRST 1 (loom_standby)'") + self.command(*psql, "SELECT pg_reload_conf()") + standby_data = self.folder / "standby-data" + standby_data.mkdir(mode=0o777) + standby_data.chmod(0o777) + standby_script = self.folder / "standby.sh" + standby_script.write_text("""#!/usr/bin/env bash +set -euo pipefail +export PGDATA=/var/lib/postgresql/data +if [ "$(id -u)" = 0 ]; then + chown postgres:postgres "$PGDATA" + exec gosu postgres bash "$0" +fi +pg_basebackup -h primary -U replicator -D "$PGDATA" -R -X stream +connection="host=primary user=replicator password=replicator" +echo "primary_conninfo = '$connection application_name=loom_standby'" \ + >> "$PGDATA/postgresql.auto.conf" +chmod 700 "$PGDATA" +exec postgres -D "$PGDATA" -c shared_buffers=128MB +""") + standby_script.chmod(0o644) + standby = self.create( + "pg-standby", + [ + "--network", + network_name, + "--memory", + str(GIB), + "--memory-swap", + str(GIB), + "-p", + f"127.0.0.1:{standby_port}:5432", + "-v", + str(standby_data) + ":/var/lib/postgresql/data", + "-e", + "PGPASSWORD=replicator", + "-v", + str(standby_script) + ":/standby.sh:ro", + "--entrypoint", + "bash", + "postgres:16", + "/standby.sh", + ], + ) + self.wait( + lambda: ( + self.command( + *psql, + ( + "SELECT sync_state FROM pg_stat_replication WHERE " + "application_name='loom_standby'" + ), + ) + == "sync" + ) + ) + self.wait( + lambda: ( + self.command( + "docker", + "exec", + ch["id"], + "clickhouse-client", + "--user", + "loom", + "--password", + "loomdev", + "--query", + "SELECT 1", + ) + == "1" + ) + ) + return ch, pg, standby, selected + + def observation(self, item: dict, role: str) -> dict: + actual = self.inspect(item["id"]) + require( + container_matches(actual, item, self.fixture_id) and actual["State"]["Running"], + "Actual job-owned running store", + ) + pid = actual["State"]["Pid"] + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + creator = self.folder / (item["role"] + "-creator.json") + mappings = [ + { + "host": value["HostIp"], + "host_port": int(value["HostPort"]), + "internal_port": int(key.split("/")[0]), + "protocol": key.split("/")[1], + } + for key, values in actual["NetworkSettings"]["Ports"].items() + for value in values or [] + ] + disk_path = ( + self.folder + / { + "clickhouse": "clickhouse-data", + "pg_primary": "pg-data", + "pg_standby": "standby-data", + }[role] + ) + return { + "role": role, + "container_id": item["id"], + "container_name": actual["Name"].lstrip("/"), + "created_at": actual["Created"], + "owner_evidence": {"path": str(creator), "sha256": sha(creator)}, + "host_pid": pid, + "start_ticks": fields[19], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": Path("/proc", str(pid)).stat().st_uid, + "ports": mappings, + "memory_limit_bytes": actual["HostConfig"]["Memory"], + "server_limit_bytes": 9 * GIB // 2 if role == "clickhouse" else 0, + "shared_buffers_bytes": 128 * 1024**2 if role != "clickhouse" else 0, + "retained": False, + "disk_path": str(disk_path), + } + + def cleanup(self) -> list[str]: + errors = [] + for item in reversed(self.items): + try: + if item["id"] is not None and not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + item["removed"] = True + self.save() + continue + if item.get("removed"): + require( + not self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed container reappeared", + ) + continue + # Unique name plus exact random label resolves a partial create only. + actual = self.inspect(item["id"] or item["name"], cleanup=True) + if item["id"] is None: + require( + actual["Name"] == "/" + item["name"] + and actual["Config"]["Labels"].get("loom.ci.trigger") == self.fixture_id, + "Partial create belongs to this job", + ) + item.update(id=actual["Id"], created=actual["Created"]) + require( + container_matches(actual, item, self.fixture_id), "Cleanup ownership changed" + ) + self.command("docker", "rm", "--force", "--volumes", item["id"], cleanup=True) + remaining = self.command( + "docker", + "ps", + "-a", + "--no-trunc", + "--filter", + "id=" + item["id"], + "--format", + "{{.ID}}", + cleanup=True, + ) + require(not remaining, "Owned container remains after cleanup") + item["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":" + item["role"]) + if self.network is not None: + try: + if self.network["id"] is not None and not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ): + self.network["removed"] = True + self.save() + if self.network.get("removed"): + require( + not self.command( + "docker", + "network", + "ls", + "--no-trunc", + "--filter", + "id=" + self.network["id"], + "--format", + "{{.ID}}", + cleanup=True, + ), + "Previously removed network reappeared", + ) + else: + rows = json.loads( + self.command( + "docker", + "network", + "inspect", + self.network["id"] or self.network["name"], + cleanup=True, + ) + ) + require( + len(rows) == 1 + and rows[0]["Labels"].get("loom.ci.trigger") == self.fixture_id + and (self.network["id"] is None or rows[0]["Id"] == self.network["id"]), + "Network cleanup ownership changed", + ) + self.network["id"] = rows[0]["Id"] + self.command("docker", "network", "rm", rows[0]["Id"], cleanup=True) + self.network["removed"] = True + self.save() + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + errors.append(type(error).__name__ + ":network") + write( + self.folder / "store-cleanup.json", + { + "removed": not errors, + "errors": errors, + "containers": [{"id": item["id"], "role": item["role"]} for item in self.items], + }, + replace=True, + ) + return errors + + +def dependency_receipt(root: Path, output: Path) -> dict: + # The CI job uses the same ARM64 CPython ABI as the reviewed codec contract. + sys.path.insert(0, str(SUPPORT)) + from fixture import CODEC_FILES, CODEC_VERSIONS + + value = { + "schema": 2, + "root": str(root), + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "versions": CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(root / ".venv" / name), "sha256": sha(root / ".venv" / name)} + for name in CODEC_FILES + }, + } + write(output, value) + return {"path": str(output), "sha256": sha(output)} + + +def publish(folder: Path, destination: Path, error: BaseException | None) -> None: + destination.mkdir(mode=0o700) + # Raw native logs/settings contain bearer and bootstrap credentials: never upload them. + fixture = folder / "fixture" + for name in ( + "outer-result.json", + "driver.jsonl", + "provenance.json", + "final-retirement.json", + "teardown.json", + ): + path = fixture / name + if path.is_file(): + shutil.copyfile(path, destination / name) + for name in ("source-before.json", "source-after.json"): + if (folder / name).is_file(): + shutil.copyfile(folder / name, destination / name) + intent_output = destination / "delivery-intent" + for arm in range(10): + for suffix in (".jsonl", "-receipt.json"): + path = fixture / "delivery-intent" / (f"arm-{arm}" + suffix) + if path.is_file() and not path.is_symlink(): + intent_output.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(path, intent_output / path.name) + if (folder / "store-cleanup.json").exists(): + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + write( + destination / "result.json", + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + "utc": datetime.datetime.now(datetime.UTC).isoformat(), + }, + ) + + +def cleanup_only(root: Path, run: dict) -> int: + state = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json" + if not state.exists(): + return 0 # The creator never published an intent and could not create a store. + require( + not state.is_symlink() + and state.stat().st_uid == os.geteuid() + and state.stat().st_size < 65536, + "Owned CI cleanup pointer", + ) + pointer = json.loads(state.read_text()) + fixture_id = pointer["fixture_id"] + require(str(uuid.UUID(fixture_id)) == fixture_id, "Cleanup fixture UUID") + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + require( + folder.resolve() == folder + and pointer == {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + "Cleanup pointer belongs to this exact CI attempt", + ) + ledger = folder / "stores.json" + require( + not ledger.is_symlink() + and ledger.stat().st_uid == os.geteuid() + and ledger.stat().st_size < 65536, + "Owned creator ledger", + ) + data = json.loads(ledger.read_text()) + require(data["fixture_id"] == fixture_id and data["ci_run"] == run, "Current creator run") + stores = Stores(root, folder, fixture_id, run) + roles = {"clickhouse", "pg-primary", "pg-standby"} + require( + len(data["items"]) <= 3 + and len({item["role"] for item in data["items"]}) == len(data["items"]), + "Bounded exact creator roles", + ) + for item in data["items"]: + require( + item["role"] in roles + and item["name"] == stores.prefix + "-" + item["role"] + and (item["id"] is None or re.fullmatch(r"[a-f0-9]{64}", item["id"])), + "Exact recorded container identity", + ) + network = data["network"] + require(network is None or network["name"] == stores.prefix + "-net", "Exact recorded network") + stores.items, stores.network = data["items"], network + native_cleanup_complete(folder, fixture_id) + errors = stores.cleanup() + destination = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + destination.mkdir(mode=0o700, exist_ok=True) + shutil.copyfile(folder / "store-cleanup.json", destination / "store-cleanup.json") + return int(bool(errors)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--clickhouse-image") + parser.add_argument("--cleanup-only", action="store_true") + args = parser.parse_args() + require( + os.environ.get("GITHUB_ACTIONS") == os.environ.get("CI") == "true", "GitHub Actions only" + ) + root = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + run = { + "run_id": os.environ["GITHUB_RUN_ID"], + "attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "repository": os.environ["GITHUB_REPOSITORY"], + } + if args.cleanup_only: + return cleanup_only(root, run) + require(args.clickhouse_image, "Resolved ClickHouse release image") + require( + Path(sys.executable) == root / ".venv/bin/python" and sys.version_info[:2] == (3, 12), + "Use the locked checkout Python 3.12", + ) + before = source(root) + resources(Path(os.environ["RUNNER_TEMP"])) + fixture_id = str(uuid.uuid4()) + folder = Path(os.environ["RUNNER_TEMP"]) / ("cell-trigger-" + fixture_id) + folder.mkdir(mode=0o700) + os.umask(0o077) + hashes_before = source_hashes(root) + write(folder / "source-before.json", {**before, "source_hashes": hashes_before, "ci_run": run}) + stores = Stores(root, folder, fixture_id, run) + stores.save() + write( + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-state.json", + {"fixture_id": fixture_id, "folder": str(folder), "ci_run": run}, + ) + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + error = None + cleanup_complete = False + try: + ch, primary, standby, selected = stores.start(args.clickhouse_image) + resources(folder) + dependencies = dependency_receipt(root, folder / "dependencies.json") + from fixture import runtime + + spec = { + "schema": 2, + "fixture_id": fixture_id, + "root": str(root), + **before, + "python": str(root / ".venv/bin/python"), + "python_sha256": sha(root / ".venv/bin/python"), + "lockfile_sha256": sha(root / "uv.lock"), + "helper_path": str(SUPPORT / "owned_children.py"), + "source_hashes": hashes_before, + "workdir": str(folder / "fixture"), + "ports": dict(zip(("api", "router", "probe", "tls"), selected[4:], strict=True)), + "databases": { + name: "trigger_" + fixture_id.replace("-", "") + "_" + str(index) + for index, name in enumerate( + ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry") + ) + }, + "stores": { + role: stores.observation(item, role) + for item, role in ( + (ch, "clickhouse"), + (primary, "pg_primary"), + (standby, "pg_standby"), + ) + }, + "pg_admin_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[2]}/postgres", + "pg_standby_dsn": f"postgresql://loom:loomdev@127.0.0.1:{selected[3]}/postgres", + "ch": { + "host": "127.0.0.1", + "port": selected[0], + "username": "loom", + "password": "loomdev", + "keeper_prefix": "/keeper_map_tables", + }, + "dependency_receipt": dependencies, + "ci_run": run, + } + write(folder / "spec.json", spec) + require(not stores.cancelled, "Cancelled before native acceptance") + require(runtime(spec) == 0, "Trigger acceptance did not pass") + require(source(root) == before, "Acceptance source changed") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught + finally: + # Native ownership closes first; exact created containers are removed even on failure. + for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): + signal.signal(signum, stores.interrupted) + try: + native_cleanup_complete(folder, fixture_id) + cleanup_errors = stores.cleanup() + cleanup_complete = not cleanup_errors + if cleanup_errors and error is None: + error = RuntimeError("Owned store cleanup incomplete") + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted") + try: + require(source(root) == before, "Final acceptance source changed") + hashes_after = source_hashes(root) + write( + folder / "source-after.json", + {**before, "source_hashes": hashes_after, "ci_run": run}, + ) + require( + hashes_after == hashes_before, "Final driver, support or workflow bytes changed" + ) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + try: + publish(folder, Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", error) + except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + error = caught if error is None else error + stores.terminal = cleanup_complete + if stores.cancelled and error is None: + error = InterruptedError("Acceptance interrupted during publication") + print( + json.dumps( + { + "accepted": error is None, + "error_type": type(error).__name__ if error is not None else "", + "failure": failure_site(error), + } + ) + ) + return int(error is not None or bool(stores.cancelled)) + + +if __name__ == "__main__": + try: + code = main() + except BaseException as error: # noqa: BLE001 — preflight failures must emit only nonsecret diagnostics. + result = { + "accepted": False, + "error_type": type(error).__name__, + "failure": failure_site(error), + } + print(json.dumps(result), flush=True) + if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("RUNNER_TEMP"): + output = Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts" + output.mkdir(mode=0o700, exist_ok=True) + if not (output / "result.json").exists(): + write(output / "result.json", result) + code = 1 + raise SystemExit(code) diff --git a/scripts/gates/testsupport/cell_trigger/__init__.py b/scripts/gates/testsupport/cell_trigger/__init__.py new file mode 100644 index 0000000000..e5dad64539 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/__init__.py @@ -0,0 +1 @@ +"""Isolated native trigger acceptance support.""" diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py new file mode 100644 index 0000000000..1d398f00da --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -0,0 +1,1024 @@ +"""Capture unchanged delivery intent and independently verify persisted rows.""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import importlib.util +import json +import math +import os +import re +import socket +import struct +import sys +import uuid +from pathlib import Path + +from fixture import ( + FILES, + HERE, + PROTOCOL, + PROTOCOL_SHA, + SPEC_FIELDS, + digest, + json_object, + need, + private_stream, + read, + ref, + sha, + source, + validate, + write, +) + +LINE = 65536 +ARM_BYTES = 32 * 1024 * 1024 +TOTAL_BYTES = 320 * 1024 * 1024 +ORDER = (True, False, False, True, True, False, False, True, True, False) +SUBJECT = "process:benchmark" + + +def protocol() -> dict: + return read(HERE / PROTOCOL, expected_sha256=PROTOCOL_SHA) + + +def integer(value, low=0, high=2**32 - 1): + need(type(value) is int and low <= value <= high, "typed bounded integer") + return value + + +def identifier(value): + if isinstance(value, uuid.UUID): + value = str(value) + need( + type(value) is str and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0, + "canonical UUID", + ) + return value + + +def equal(left, right): + need( + json.dumps(left, sort_keys=True, allow_nan=False, separators=(",", ":")) + == json.dumps(right, sort_keys=True, allow_nan=False, separators=(",", ":")), + "exact typed equality", + ) + + +def instant(value, *, audit=False): + if isinstance(value, str): + need( + re.fullmatch( + r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d:\d\d)?", value + ), + "exact ISO datetime", + ) + value = datetime.datetime.fromisoformat(value) + need(type(value) is datetime.datetime, "typed datetime") + value = ( + value.replace(tzinfo=datetime.UTC) + if value.tzinfo is None + else value.astimezone(datetime.UTC) + ) + if audit: + need(value.microsecond % 1000 == 0, "audit millisecond precision") + return value.isoformat(timespec="microseconds") + need(value.microsecond == 0 and 0 <= value.timestamp() < 2**32, "memory second precision/range") + return value.replace(tzinfo=None).isoformat(timespec="seconds") + + +def hex64(value, *, native=False): + if native and isinstance(value, (bytes, bytearray)): + value = bytes(value).decode("ascii") + need(type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value), "typed hex64") + return value + + +def payload(key): + prefix = f"Cell trigger measurement {key}: " + need( + type(key) is str and key.isascii() and len(prefix) < 4096 and len(key) <= 128, "payload key" + ) + return prefix + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(prefix)] + + +def content_hash(key): + return hashlib.sha256( + json.dumps(payload(key), sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +class Codec: + def __init__(self, specification): + self.p = specification + need( + len(self.p["memory_fields"]) == 61 + and len(set(self.p["memory_fields"])) == 61 + and set(self.p["memory_types"]) == set(self.p["memory_fields"]) + and len(self.p["sparse_defaults"]) == 43 + and len(self.p["audit_fields"]) == 13, + "closed protocol", + ) + + def row(self, original, *, origin="capture", comparison=False): + need( + type(original) is dict and set(original) <= set(self.p["memory_fields"]), + "memory row fields", + ) + value = dict(original) + if origin == "pg": + for key, default in self.p["sparse_defaults"].items(): + if key not in value: + value[key] = list(default) if isinstance(default, list) else default + need(set(value) == set(self.p["memory_fields"]), "complete memory row") + result = {} + for key in self.p["memory_fields"]: + item = value[key] + kind = self.p["memory_types"][key] + if kind.startswith("nullable_") and item is None: + result[key] = None + continue + kind = kind.removeprefix("nullable_") + if kind == "string": + need(type(item) is str and len(item.encode()) <= 8192, "memory string") + elif kind == "json_object_field": + if origin == "ch": + need(type(item) is dict, "native CH JSON object") + parsed = json_object(json.dumps(item, allow_nan=False)) + else: + need(type(item) is str, "encoded object JSON") + parsed = json_object(item) + need(parsed == {}, "primitive workload JSON object") + if comparison: + item = parsed + elif kind == "utc_second": + item = instant(item) + elif kind == "uuid": + item = identifier(item) + elif kind in {"array_uuid", "array_string"}: + need(type(item) is list and len(item) <= 64, "typed array") + if kind == "array_uuid": + item = [identifier(member) for member in item] + else: + need( + all(type(member) is str and len(member.encode()) <= 512 for member in item), + "string array", + ) + elif kind == "hex64_fixedstring": + item = hex64(item, native=origin == "ch") + elif kind == "finite_exact_float32_not_bool": + need(type(item) in {int, float} and math.isfinite(item), "finite Float32") + rounded = struct.unpack("!f", struct.pack("!f", item))[0] + need(rounded == item, "exact stored Float32") + item = rounded + elif kind == "positive_int63_not_bool": + item = integer(item, 1, 2**63 - 1) + elif kind == "uint8_0_or_1_not_bool": + item = integer(item, 0, 1) + elif kind == "uint32_not_bool": + item = integer(item) + else: + need(False, "unknown protocol type") + result[key] = item + return result + + def audit(self, value, *, origin="capture"): + need( + type(value) is dict and set(value) == set(self.p["audit_fields"]), + "complete audit fields", + ) + result = {} + for key in self.p["audit_fields"]: + item = value[key] + if key == "ts": + item = instant(item, audit=True) + elif key == "row_hash": + item = hex64(item, native=origin == "ch") + else: + need(type(item) is str and len(item.encode()) <= 8192, "audit string") + result[key] = item + timestamp = ( + datetime.datetime.fromisoformat(result["ts"]) + .astimezone(datetime.UTC) + .replace(tzinfo=None) + .isoformat() + ) + canonical = { + key: result[key] + for key in ("event_type", "org", "namespace", "actor", "subject", "request_id") + } + canonical["ts"] = timestamp + actual = hashlib.sha256( + json.dumps(canonical, sort_keys=True, ensure_ascii=True).encode() + ).hexdigest() + need(actual == result["row_hash"], "canonical raw-subject audit hash") + return result + + def expected(self, row, audit, context, *, api, metadata, driver): + key = row["memory_key"] + equal( + [ + row[k] + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "actor", + "source_class", + ) + ], + [context["org"], context["namespace"], "bench", "-", "-", SUBJECT, "asserted"], + ) + need( + row["memory_type"] == "semantic" and row["status"] == "active" and row["pinned"] == 0, + "workload semantic/active", + ) + need(row["confidence"] == 1.0, "workload confidence") + for field, default in self.p["sparse_defaults"].items(): + equal(row[field], default) + need( + row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), + "exact stored payload/hash", + ) + expected_subject = dict( + memory_key=key, + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + source_class="asserted", + status="active", + pinned=False, + tombstoned=False, + ) + equal(json_object(audit["subject"]), expected_subject) + need( + audit["subject"] == json.dumps(expected_subject, sort_keys=True), + "raw native subject spelling", + ) + need( + audit["event_type"] == "memory.revision" + and audit["org"] == context["org"] + and audit["namespace"] == context["namespace"] + and audit["actor"] == SUBJECT, + "audit event/scope/actor", + ) + api_launch = context["_api_server"] if api else None + instance = ( + api_launch["health_instance"] if api else driver["host"] + ":" + str(driver["pid"]) + ) + equal( + { + k: audit[k] + for k in ("build_commit", "instance", "instance_label", "consumer", "token_id") + }, + dict( + build_commit=context["head"], + instance=instance, + instance_label=("iso:trigger:" + context["fixture_id"][:12])[:64], + consumer="service-account" if api else "", + token_id=metadata["summary"]["jti"] if api else "", + ), + ) + need( + ( + type(audit["request_id"]) is str + and audit["request_id"].isascii() + and 0 < len(audit["request_id"]) <= 128 + ) + if api + else audit["request_id"] == "", + "audit request identity", + ) + + +def birth(): + pid = os.getpid() + return dict( + pid=pid, + start_ticks=Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[19], + boot_id=Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + uid=os.geteuid(), + host=socket.gethostname(), + environment_sha256=digest(dict(os.environ)), + ) + + +def line(value): + raw = ( + json.dumps(value, sort_keys=True, allow_nan=False, separators=(",", ":")) + "\n" + ).encode() + need(len(raw) <= LINE, "JSONL line bound") + return raw + + +class Capture: + def __init__(self, context, directory, driver_birth, codec): + self.s = context + self.directory = directory + self.birth = driver_birth + self.codec = codec + self.calls = 0 + self.total = 0 + need( + directory == Path(context["workdir"]) / "delivery-intent" and not directory.exists(), + "new exact intent directory", + ) + directory.mkdir(mode=0o700) + + def header(self, arm): + w = Path(self.s["workdir"]) + return dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=self.s["capture_context_sha256"], + fixture_sha256=sha(w / "fixture.json"), + launch_sha256=sha(w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + birth=self.birth, + ) + + def delegate(self, original): + async def build(args, batch): + arm = self.calls + need(arm < 10 and type(batch) is int and batch == 500, "exact build count/batch") + self.calls += 1 + block = await original(args, batch) + need(len(block.rows) == len(block.audits) == 2500, "exact delivery intent count") + path = self.directory / f"arm-{arm}.jsonl" + full = hashlib.sha256() + prefix = hashlib.sha256() + size = 0 + header = self.header(arm) + with private_stream(path, output=True) as stream: + for index in range(-1, 2501): + if index == -1: + value = header + elif index == 2500: + value = dict( + schema=1, + kind="trailer", + arm=arm, + count=2500, + prefix_sha256=prefix.hexdigest(), + ) + else: + value = dict( + schema=1, + kind="delivery", + arm=arm, + index=index, + row=self.codec.row(block.rows[index]), + audit=self.codec.audit(block.audits[index]), + ) + raw = line(value) + size += len(raw) + self.total += len(raw) + need(size <= ARM_BYTES and self.total <= TOTAL_BYTES, "intent byte bounds") + need(stream.write(raw) == len(raw), "complete intent write") + full.update(raw) + if index < 2500: + prefix.update(raw) + write( + self.directory / f"arm-{arm}-receipt.json", + dict( + schema=1, + arm=arm, + bytes=size, + records=2502, + sha256=full.hexdigest(), + header=header, + ), + ) + return block + + return build + + +def mint_metadata(context, minted): + w = Path(context["workdir"]) + with private_stream(w / "mint-summary.stdout") as stream: + raw = stream.read(4097) + need(len(raw) <= 4096 and raw.endswith(b"\n") and raw.count(b"\n") == 1, "mint stdout shape") + summary = json_object(raw) + need(set(summary) == set(protocol()["mint_summary_fields"]), "mint summary fields") + need(type(summary["jti"]) is str and re.fullmatch(r"[a-f0-9]{32}", summary["jti"]), "mint jti") + equal( + {k: v for k, v in summary.items() if k != "jti"}, + dict( + subject="benchmark", + org=context["org"], + namespaces=[context["namespace"]], + roles=["writer"], + ttl_days=1, + is_service_account=True, + read_scope="namespace", + read_user_id="", + traffic_class="", + consumer="", + token_path=str(w / "writer.jwt"), + ), + ) + with private_stream(w / "writer.jwt") as stream: + token = stream.read(16385) + need( + 0 < len(token) <= 16384 and token.endswith(b"\n") and token.count(b"\n") == 1, + "private native bearer file", + ) + born = ref(minted["launch_receipt"]) + terminal = ref(minted["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] == 0, "mint terminal success" + ) + equal(minted["source_sha256"], digest(context["source_hashes"])) + equal(minted["cwd"], context["root"]) + expected = [ + context["python"], + "-m", + "server.cli", + "mint-token", + "--subject", + "benchmark", + "--org", + context["org"], + "--namespaces", + context["namespace"], + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(w / "writer.jwt"), + "--json", + ] + equal(minted["redacted_argv"], expected) + for key in ("pid", "start_ticks", "boot_id", "uid", "configuration_sha256"): + equal(minted[key], born[key]) + retirement = read(w / "mint-proof.json") + equal(retirement["birth"], born) + need(retirement["state"] == "retired", "mint retirement receipt") + return dict( + schema=1, + summary=summary, + stdout=dict( + path=str(w / "mint-summary.stdout"), + sha256=hashlib.sha256(raw).hexdigest(), + bytes=len(raw), + ), + token=dict( + path=str(w / "writer.jwt"), sha256=hashlib.sha256(token).hexdigest(), bytes=len(token) + ), + process=minted, + retirement=dict(path=str(w / "mint-proof.json"), sha256=sha(w / "mint-proof.json")), + ), token.decode().strip() + + +def verify_token(metadata): + expected = metadata["token"] + with private_stream(Path(expected["path"])) as stream: + raw = stream.read(16385) + need( + len(raw) == expected["bytes"] and hashlib.sha256(raw).hexdigest() == expected["sha256"], + "same actual native bearer bytes", + ) + return raw.decode().strip() + + +def closed(value, fields): + need(type(value) is dict and set(value) == set(fields.split()), "closed artifact record") + + +def record_stream(path, *, maximum): + full = hashlib.sha256() + size = 0 + count = 0 + with private_stream(path) as stream: + while True: + raw = stream.readline(LINE + 1) + if not raw: + break + count += 1 + size += len(raw) + need( + len(raw) <= LINE and raw.endswith(b"\n") and size <= maximum and count <= 101000, + "bounded complete JSONL", + ) + full.update(raw) + yield json_object(raw), raw + return dict(path=str(path), sha256=full.hexdigest(), bytes=size, records=count) + + +class Workload: + def __init__(self, context, codec): + self.s = context + self.w = Path(context["workdir"]) + self.codec = codec + self.metadata = ref(context["mint_metadata"]) + verified, _ = mint_metadata(context, self.metadata["process"]) + equal(self.metadata, verified) + verify_token(self.metadata) + self.driver = read(self.w / "driver-birth.json") + self.driver_birth = ref(self.driver["launch_receipt"]) + self.finished = read(self.w / "driver-finished.json") + terminal = ref(self.finished["terminal_receipt"]) + need( + type(terminal["returncode"]) is int and terminal["returncode"] in (0, 1), + "completed actual driver status", + ) + for key in ( + "pid", + "start_ticks", + "boot_id", + "uid", + "configuration_sha256", + "launch_receipt", + ): + equal(self.driver[key], self.finished[key]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.driver[key], self.driver_birth[key]) + equal(self.driver["source_sha256"], digest(context["source_hashes"])) + equal(self.driver["cwd"], context["root"]) + self.launch = read(self.w / "serving-cell/launch.json") + self.configuration = ref(self.launch["configuration_artifact"]) + fixture = read(self.w / "fixture.json") + ownership = read( + self.w / "ownership.json", expected_sha256=fixture["ownership_receipt_sha256"] + ) + candidates = [ + item for item in ownership["processes"] if item["pid"] == self.launch["server"]["pid"] + ] + need(len(candidates) == 1, "one owner-recorded actual API birth") + actual = ref(candidates[0]["launch_receipt"]) + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(self.launch["server"][key], actual[key]) + equal(self.launch["source"], dict(head=context["head"], tree=context["tree"])) + equal(self.launch["server"]["instance_id"], context["api_instance_id"]) + equal(self.launch["server"]["cwd"], context["root"]) + equal( + self.launch["server"]["health_instance"], + socket.gethostname() + ":" + str(self.launch["server"]["pid"]), + ) + self.ids = set() + self.keys = set() + self.total = 0 + self.counts = {"api": 0, "delivery": 0} + self.refs = [ + dict(path=str(self.w / name), sha256=sha(self.w / name)) + for name in ( + "fixture.json", + "attached-context.json", + "ownership.json", + "serving-cell/launch.json", + "driver-birth.json", + "driver-finished.json", + ) + ] + self.refs.extend( + [ + self.launch["configuration_artifact"], + context["mint_metadata"], + self.metadata["stdout"], + self.metadata["token"], + self.metadata["process"]["launch_receipt"], + self.metadata["process"]["terminal_receipt"], + self.metadata["retirement"], + ] + ) + self.driver_header = None + self.iterator = None + + def __enter__(self): + self.iterator = self.inputs() + return self + + def __exit__(self, kind, error, traceback): + try: + if self.iterator is not None: + self.iterator.close() + except BaseException: + if error is None: + raise + return False + + def unique(self, item): + identifier(item["memory_id"]) + key = item["key"] + need( + item["memory_id"] not in self.ids and key not in self.keys, + "unique independent workload identities", + ) + self.ids.add(item["memory_id"]) + self.keys.add(key) + need(len(self.ids) <= 45500 and len(self.keys) <= 45500, "workload set bound") + self.counts[item["kind"]] += 1 + return item + + def delivery(self, arm): + path = self.w / "delivery-intent" / f"arm-{arm}.jsonl" + receipt = read(path.parent / f"arm-{arm}-receipt.json") + closed(receipt, "schema arm bytes records sha256 header") + equal([receipt[key] for key in ("schema", "arm", "records")], [1, arm, 2502]) + integer(receipt["bytes"], 1, ARM_BYTES) + hex64(receipt["sha256"]) + prefix = hashlib.sha256() + full = hashlib.sha256() + size = 0 + records = record_stream(path, maximum=ARM_BYTES) + try: + header, raw = next(records) + full.update(raw) + prefix.update(raw) + size += len(raw) + closed( + header, + ( + "schema kind fixture_id arm on batch_rows expected_rows head " + "tree driver_sha256 capture_sha256 protocol_sha256 " + "context_sha256 fixture_sha256 launch_sha256 " + "mint_metadata_sha256 birth" + ), + ) + equal(header, receipt["header"]) + for key, expected in dict( + schema=1, + kind="header", + fixture_id=self.s["fixture_id"], + arm=arm, + on=ORDER[arm], + batch_rows=500, + expected_rows=2500, + head=self.s["head"], + tree=self.s["tree"], + driver_sha256=self.s["source_hashes"]["driver"], + capture_sha256=self.s["source_hashes"][FILES[3]], + protocol_sha256=PROTOCOL_SHA, + context_sha256=sha(self.w / "attached-context.json"), + fixture_sha256=sha(self.w / "fixture.json"), + launch_sha256=sha(self.w / "serving-cell/launch.json"), + mint_metadata_sha256=self.s["mint_metadata"]["sha256"], + ).items(): + equal(header[key], expected) + born = header["birth"] + closed(born, "pid start_ticks boot_id uid host environment_sha256") + for key in ("pid", "start_ticks", "boot_id", "uid"): + equal(born[key], self.driver[key]) + need( + born["host"] == socket.gethostname() + and born["environment_sha256"] == self.driver["configuration_sha256"], + "actual driver capture birth/config", + ) + if self.driver_header is None: + self.driver_header = born + else: + equal(born, self.driver_header) + pattern = None + for index in range(2500): + value, raw = next(records) + size += len(raw) + full.update(raw) + prefix.update(raw) + closed(value, "schema kind arm index row audit") + equal( + [value[k] for k in ("schema", "kind", "arm", "index")], + [1, "delivery", arm, index], + ) + row = self.codec.row(value["row"]) + audit = self.codec.audit(value["audit"]) + match = re.fullmatch(r"(gate-delivery-[a-f0-9]{32})-(\d+)", row["memory_key"]) + need(match is not None and match[2] == str(index), "delivery exact key counter") + if pattern is None: + pattern = match[1] + need(pattern == match[1], "one original delivery prefix") + yield self.unique( + dict( + kind="delivery", + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + row=row, + audit=audit, + ) + ) + trailer, raw = next(records) + size += len(raw) + full.update(raw) + equal( + trailer, + dict( + schema=1, kind="trailer", arm=arm, count=2500, prefix_sha256=prefix.hexdigest() + ), + ) + try: + next(records) + except StopIteration: + pass + else: + need(False, "extra intent record") + need( + receipt["bytes"] == size and receipt["sha256"] == full.hexdigest(), + "complete intent bytes", + ) + self.total += size + need(self.total <= TOTAL_BYTES, "intent aggregate") + self.refs.append(dict(path=str(path), sha256=full.hexdigest(), bytes=size)) + self.refs.append( + dict( + path=str(path.parent / f"arm-{arm}-receipt.json"), + sha256=sha(path.parent / f"arm-{arm}-receipt.json"), + ) + ) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def inputs(self): + records = record_stream(self.w / "driver.jsonl", maximum=64 * 1024 * 1024) + + def take(kind, fields=None): + value, _ = next(records) + need(value.get("kind") == kind, "original artifact event order") + if fields is not None: + closed(value, fields) + return value + + try: + config = take( + "configuration", + ( + "kind manifest shared_configuration_excludes " + "credential_posture api_configuration_attested source launch " + "manifest_sha256 calls warmup rounds batches batch_rows " + "logical_keys value_bytes order driver_sha256" + ), + ) + equal(config["manifest"], read(self.w / "fixture.json")) + need(config["manifest_sha256"] == digest(config["manifest"]), "artifact fixture digest") + equal(config["source"], dict(head=self.s["head"], tree=self.s["tree"], clean=True)) + equal( + config["launch"], + dict( + health_instance=self.launch["server"]["health_instance"], + fingerprints=self.configuration["fingerprints"], + launch_sha256=sha(self.w / "serving-cell/launch.json"), + configuration_artifact_sha256=self.launch["configuration_artifact"]["sha256"], + birth_sha256=digest(self.launch["server"]), + ), + ) + equal(config["credential_posture"], self.configuration["credential_posture"]) + for key in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(config["manifest"][key], self.configuration[key]) + for key, expected in dict( + calls=2000, + warmup=50, + rounds=5, + batches=5, + batch_rows=500, + logical_keys=45500, + value_bytes=4096, + order=list(ORDER), + driver_sha256=self.s["source_hashes"]["driver"], + shared_configuration_excludes=["cell_router_token"], + api_configuration_attested=True, + ).items(): + equal(config[key], expected) + for arm in range(10): + proof = take( + "arm_proof", + ( + "kind arm on deployment_id epoch instance_id " + "fingerprint_sha256 api_configuration_sha256 " + "cli_configuration_sha256 shared_configuration_sha256 " + "shared_configuration_excludes credential_posture " + "api_configuration_attested source launch tables " + "invariant_sha256 org_triggers_sha256" + ), + ) + equal([proof["arm"], proof["on"]], [arm, ORDER[arm]]) + for key in ( + "source", + "launch", + "credential_posture", + "shared_configuration_excludes", + "api_configuration_attested", + ): + equal(proof[key], config[key]) + for key in ( + "deployment_id", + "epoch", + "instance_id", + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ): + equal(proof[key], config["manifest"][key]) + pattern = None + for index in range(2050): + value = take( + "request", + ( + "kind arm index status key memory_id version content_sha256 " + "elapsed_ms measured admission" + ), + ) + equal( + [value[k] for k in ("arm", "index", "status", "measured", "admission")], + [arm, index, 200, index >= 50, "derived_ready"], + ) + need( + type(value["elapsed_ms"]) in {int, float} + and math.isfinite(value["elapsed_ms"]) + and value["elapsed_ms"] >= 0, + "finite recorded latency", + ) + integer(value["version"], 1, 2**63 - 1) + match = re.fullmatch( + r"(gate-[a-f0-9]{32}-" + str(arm) + r")-(\d+)", value["key"] + ) + need(match is not None and match[2] == str(index), "API exact key counter") + if pattern is None: + pattern = match[1] + need( + match[1] == pattern + and value["content_sha256"] == content_hash(value["key"]), + "API original prefix/hash", + ) + yield self.unique( + dict( + kind="api", + **{ + key: value[key] + for key in ("key", "memory_id", "version", "content_sha256") + }, + ) + ) + for index in range(5): + value = take( + "worker", + ( + "kind arm index claimed published failed expired_keys " + "expired_evidence swept_settled published_audits" + ), + ) + equal( + [value[k] for k in ("arm", "index", "claimed", "published", "failed")], + [arm, index, 500, 500, 0], + ) + for key in ( + "expired_keys", + "expired_evidence", + "swept_settled", + "published_audits", + ): + integer(value[key]) + value = take("delivery_block", "kind arm elapsed_seconds rows rps") + equal([value["arm"], value["rows"]], [arm, 2500]) + need( + type(value["elapsed_seconds"]) in {int, float} + and math.isfinite(value["elapsed_seconds"]) + and value["elapsed_seconds"] > 0 + and type(value["rps"]) in {int, float} + and math.isfinite(value["rps"]) + and value["rps"] > 0, + "finite worker record", + ) + value = take("settled", "kind arm api_rows delivery_rows physical") + equal([value["arm"], value["api_rows"], value["delivery_rows"]], [arm, 2050, 2500]) + value = take("arm", "kind arm on p50_ms p99_ms delivery_rps") + equal([value["arm"], value["on"]], [arm, ORDER[arm]]) + yield from self.delivery(arm) + take( + "provisional_statistics", + ( + "kind on off p99_delta_ms throughput_loss " + "latency_within_limit throughput_within_limit " + "paired_p99_differences_ms" + ), + ) + equal( + take("triggers_restored", "kind verified"), + dict(kind="triggers_restored", verified=True), + ) + equal( + take("driver_cleanup", "kind native_returned outer_database_cleanup_required"), + dict( + kind="driver_cleanup", + native_returned=True, + outer_database_cleanup_required=True, + ), + ) + try: + next(records) + except StopIteration as end: + self.refs.append(end.value) + else: + need(False, "extra driver event") + equal(self.counts, dict(api=20500, delivery=25000)) + verify_token(self.metadata) + finally: + pending = sys.exception() + try: + records.close() + except BaseException: + if pending is None: + raise + + def pages(self): + page = [] + for value in self.iterator: + page.append(value) + if len(page) == 100: + yield page + page = [] + if page: + yield page + + +def entry(context, directory, argv): + need( + set(context) + == SPEC_FIELDS + | { + "cell", + "org", + "namespace", + "api_instance_id", + "mint_metadata", + "capture_context_sha256", + }, + "capture context fields", + ) + validate({key: context[key] for key in SPEC_FIELDS}) + source(context) + need( + len(argv) == 16 + and len(set(argv[::2])) == 8 + and set(argv[::2]) + == { + "--base-url", + "--pg-dsn", + "--token", + "--org", + "--namespace", + "--fixture-manifest", + "--launch-manifest", + "--output", + }, + "original driver arguments", + ) + # There are eight required original pairs; no private option reaches its parser. + driver_path = Path(context["root"]) / "bench/cell_trigger_gate.py" + name = "_fixture_measured_driver" + need(name not in sys.modules, "single actual driver module") + spec = importlib.util.spec_from_file_location(name, driver_path) + need(spec is not None and spec.loader is not None, "exact driver module") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + capture = Capture(context, directory, birth(), Codec(protocol())) + module._build_block = capture.delegate(module._build_block) + sys.argv = [str(driver_path), *argv] + result = module.main() + return 1 if result == 0 and capture.calls != 10 else result + + +def main(): + try: + divider = sys.argv.index("--") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--context", type=Path, required=True) + parser.add_argument("--capture-directory", type=Path, required=True) + args = parser.parse_args(sys.argv[1:divider]) + context = read(args.context) + context["capture_context_sha256"] = sha(args.context) + return entry(context, args.capture_directory, sys.argv[divider + 1 :]) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(type(error).__name__, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py new file mode 100644 index 0000000000..42526c9e43 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -0,0 +1,1250 @@ +"""Own the isolated CI trigger acceptance fixture and its cleanup.""" + +from __future__ import annotations + +import argparse +import ast +import datetime +import fcntl +import hashlib +import json +import math +import os +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import threading +import time +import uuid +from contextlib import contextmanager +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +GIB = 2**30 + +POLICY = dict( + admit_mem_bytes=10 * GIB, + admit_disk_bytes=16 * GIB, + stop_mem_bytes=4 * GIB, + stop_disk_bytes=16 * GIB, + stop_process_rss_bytes=7 * GIB // 2, + sample_seconds=1, + wall_seconds=7200, + drain_reserve_seconds=180, +) +HERE = Path(__file__).resolve().parent +FILES = ("fixture.py", "native.py", "tls_proxy.py", "capture.py", "owned_children.py") +PROTOCOL = "protocol.json" +PROTOCOL_SHA = "7223a2018c518cfc8a58977c84aa88dec5784aa741cb34245cb092abbf848812" +SPEC_FIELDS = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", +} +CODEC_FILES = ( + "pyvenv.cfg", + "lib/python3.12/site-packages/clickhouse_connect/_version.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/dynamic.py", + "lib/python3.12/site-packages/clickhouse_connect/datatypes/format.py", + "lib/python3.12/site-packages/asyncpg/_version.py", + "lib/python3.12/site-packages/asyncpg/connection.py", + "lib/python3.12/site-packages/asyncpg/protocol/protocol.cpython-312-aarch64-linux-gnu.so", + "lib/python3.12/site-packages/asyncpg/protocol/record.cpython-312-aarch64-linux-gnu.so", +) +CODEC_VERSIONS = {"clickhouse-connect": "1.9.0", "asyncpg": "0.32.0"} + + +class Refused(RuntimeError): + pass + + +def need(ok: object, field: str) -> None: + if not ok: + raise Refused(field) + + +def digest(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, default=str, separators=(",", ":")).encode() + ).hexdigest() + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def utc() -> str: + return datetime.datetime.now(datetime.UTC).isoformat() + + +def pairs(items: list[tuple]) -> dict: + out = {} + for key, value in items: + need(key not in out, "duplicate JSON field") + out[key] = value + return out + + +def invalid_constant(_value: str): + raise Refused("nonfinite JSON number") + + +def finite_float(value: str) -> float: + result = float(value) + need(math.isfinite(result), "nonfinite JSON number") + return result + + +def json_object(raw: bytes | str) -> dict: + value = json.loads( + raw, object_pairs_hook=pairs, parse_constant=invalid_constant, parse_float=finite_float + ) + + def depth(item, level=0): + need(level <= 8, "JSON nesting") + if isinstance(item, dict): + for member in item.values(): + depth(member, level + 1) + elif isinstance(item, list): + for member in item: + depth(member, level + 1) + + depth(value) + need(isinstance(value, dict), "JSON object") + return value + + +@contextmanager +def private_stream(path: Path, *, output: bool = False): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if output else os.O_RDONLY | os.O_NONBLOCK + fd = os.open(path, flags | os.O_NOFOLLOW, 0o600) + stream = None + first = None + try: + info = os.fstat(fd) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o077 == 0, + "owned private stream", + ) + stream = os.fdopen(fd, "wb" if output else "rb", closefd=False) + yield stream + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + if output: + try: + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +def read(path: Path, *, limit: int = 65536, expected_sha256: str | None = None) -> dict: + with private_stream(path) as stream: + raw = stream.read(limit + 1) + need(len(raw) <= limit, "input size") + if expected_sha256 is not None: + need(hashlib.sha256(raw).hexdigest() == expected_sha256, "receipt bytes") + return json_object(raw) + + +def write(path: Path, value: dict, *, limit: int = 65536) -> dict: + raw = (json.dumps(value, sort_keys=True, allow_nan=False) + "\n").encode() + need(len(raw) <= limit, "receipt size") + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + first = None + stream = None + try: + stream = os.fdopen(fd, "wb", closefd=False) + need(stream.write(raw) == len(raw), "complete receipt write") + stream.flush() + os.fsync(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if stream is not None: + try: + stream.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + os.close(fd) + except BaseException: + if first is None: + raise + if first is not None: + raise first + return {"path": str(path), "sha256": hashlib.sha256(raw).hexdigest()} + + +def ref(value: dict) -> dict: + need(set(value) == {"path", "sha256"}, "receipt reference fields") + path = Path(value["path"]) + need(path.is_absolute(), "absolute receipt") + need( + isinstance(value["sha256"], str) and re.fullmatch(r"[a-f0-9]{64}", value["sha256"]), + "receipt digest", + ) + return read(path, expected_sha256=value["sha256"]) + + +def source(spec: dict) -> None: + need( + sys.version_info[:2] == (3, 12) and Path(sys.executable) == Path(spec["python"]), + "actual admitted interpreter", + ) + dependencies(spec, verify=True) + root = Path(spec["root"]) + for command, expected in ( + (["rev-parse", "HEAD"], spec["head"]), + (["rev-parse", "HEAD^{tree}"], spec["tree"]), + (["status", "--porcelain"], ""), + ): + result = subprocess.run( + ["git", *command], cwd=root, capture_output=True, text=True, check=False, timeout=5 + ) + need(result.returncode == 0 and result.stdout.strip() == expected, "frozen source") + for name, expected in spec["source_hashes"].items(): + path = ( + root / "bench/cell_trigger_gate.py" + if name == "driver" + else root / "scripts/gates/cell_trigger_acceptance.py" + if name == "entrypoint" + else root / ".github/workflows/docker-smoke.yml" + if name == "workflow" + else HERE / name + ) + need(sha(path) == expected, "private source bytes") + need( + Path(spec["helper_path"]) == HERE / "owned_children.py" + and sha(Path(spec["helper_path"])) == spec["source_hashes"]["owned_children.py"], + "owned child implementation", + ) + need(sha(Path(spec["python"])) == spec["python_sha256"], "private interpreter") + need(sha(root / "uv.lock") == spec["lockfile_sha256"], "locked dependency source") + + +def dependencies(spec: dict, *, verify: bool = False) -> None: + value = ref(spec["dependency_receipt"]) + need( + set(value) + == { + "schema", + "root", + "python", + "python_sha256", + "lockfile_sha256", + "versions", + "codec_sources", + } + and type(value["schema"]) is int + and value["schema"] == 2, + "closed dependency receipt", + ) + for key in ("root", "python", "python_sha256", "lockfile_sha256"): + need(value[key] == spec[key], "dependency source identity") + environment = Path(spec["root"]) / ".venv" + need( + Path(spec["python"]) == environment / "bin/python" + and value["versions"] == CODEC_VERSIONS + and set(value["codec_sources"]) == set(CODEC_FILES), + "fixed interpreter/codec versions and file set", + ) + for name in CODEC_FILES: + item = value["codec_sources"][name] + path = environment / name + need( + set(item) == {"path", "sha256"} + and item["path"] == str(path) + and type(item["sha256"]) is str + and re.fullmatch(r"[a-f0-9]{64}", item["sha256"]), + "exact fixed codec path/hash", + ) + if verify: + info = path.lstat() + need( + stat.S_ISREG(info.st_mode) + and info.st_size <= 32 * 1024 * 1024 + and sha(path) == item["sha256"], + "regular frozen codec bytes", + ) + if verify: + cfg = dict( + line.split(" = ", 1) + for line in (environment / "pyvenv.cfg").read_text().splitlines() + if " = " in line + ) + need( + cfg.get("implementation") == "CPython" + and sys.version_info[:2] == (3, 12) + and cfg.get("version_info") in {"3.12", ".".join(map(str, sys.version_info[:3]))} + and cfg.get("include-system-site-packages") == "false", + "fixed isolated CPython environment", + ) + for package, field, name in ( + ("clickhouse-connect", "version", CODEC_FILES[1]), + ("asyncpg", "__version__", CODEC_FILES[4]), + ): + module = ast.parse((environment / name).read_bytes()) + assignments = [ + node.value + for node in module.body + if ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == field + for target in node.targets + ) + ) + or ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == field + ) + ] + need( + len(assignments) == 1 + and ast.literal_eval(assignments[0]) == CODEC_VERSIONS[package], + "frozen codec source version", + ) + + +def database(name: str) -> str: + need( + isinstance(name, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,62}", name), + "database identifier", + ) + need( + name.lower() + not in { + "loom", + "loom_router", + "loom_telemetry", + "default", + "system", + "postgres", + "template0", + "template1", + "information_schema", + }, + "scratch database", + ) + return name + + +def pg_dsn(admin: str, db: str) -> str: + value = urlsplit(admin) + need( + value.scheme in {"postgres", "postgresql"} + and value.hostname == "127.0.0.1" + and value.port + and not value.query + and not value.fragment, + "numeric PG DSN", + ) + return urlunsplit((value.scheme, value.netloc, "/" + database(db), "", "")) + + +def validate(spec: dict) -> None: + fields = { + "schema", + "fixture_id", + "root", + "head", + "tree", + "python", + "python_sha256", + "lockfile_sha256", + "helper_path", + "source_hashes", + "workdir", + "ports", + "databases", + "stores", + "pg_admin_dsn", + "pg_standby_dsn", + "ch", + "dependency_receipt", + "ci_run", + } + need(set(spec) == fields and type(spec["schema"]) is int and spec["schema"] == 2, "spec fields") + need(str(uuid.UUID(spec["fixture_id"])) == spec["fixture_id"], "fixture UUID") + for field in ("root", "python", "helper_path", "workdir"): + need(Path(spec[field]).is_absolute(), "absolute fixture path") + need( + all( + isinstance(spec[field], str) and re.fullmatch(r"[a-f0-9]{40}", spec[field]) + for field in ("head", "tree") + ), + "full source identity", + ) + need( + set(spec["source_hashes"]) == set(FILES) | {"driver", "entrypoint", "workflow", PROTOCOL}, + "source hash set", + ) + need(spec["source_hashes"][PROTOCOL] == PROTOCOL_SHA, "fixed capture protocol") + need( + all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{64}", value) + for value in spec["source_hashes"].values() + ), + "source digest values", + ) + need(set(spec["ports"]) == {"api", "router", "probe", "tls"}, "port roles") + need( + len(set(spec["ports"].values())) == 4 + and all(type(p) is int and 0 < p < 65536 for p in spec["ports"].values()), + "distinct ports", + ) + need( + set(spec["databases"]) == {"pg_cell", "pg_directory", "ch_schema", "ch_telemetry"}, + "DB roles", + ) + need(len(set(spec["databases"].values())) == 4, "distinct databases") + for name in spec["databases"].values(): + database(name) + pg_dsn(spec["pg_admin_dsn"], spec["databases"]["pg_cell"]) + need( + set(spec["ch"]) == {"host", "port", "username", "password", "keeper_prefix"}, + "CH input fields", + ) + need( + spec["ch"]["host"] == "127.0.0.1" + and type(spec["ch"]["port"]) is int + and 0 < spec["ch"]["port"] < 65536, + "numeric CH endpoint", + ) + need( + os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("CI") == "true", + "GitHub CI only", + ) + need( + spec["ci_run"] + == { + "run_id": os.environ.get("GITHUB_RUN_ID"), + "attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "repository": os.environ.get("GITHUB_REPOSITORY"), + } + and str(spec["ci_run"]["run_id"]).isdigit(), + "actual CI run binding", + ) + dependencies(spec) + need(set(spec["stores"]) == {"clickhouse", "pg_primary", "pg_standby"}, "three owned stores") + for role, expected in spec["stores"].items(): + store_fields = { + "role", + "container_id", + "container_name", + "created_at", + "owner_evidence", + "host_pid", + "start_ticks", + "boot_id", + "uid", + "ports", + "memory_limit_bytes", + "server_limit_bytes", + "shared_buffers_bytes", + "retained", + "disk_path", + } + need(set(expected) == store_fields, "store input fields") + need(expected["role"] == role and expected["retained"] is False, "job-created store") + need( + type(expected["host_pid"]) is int + and expected["host_pid"] > 1 + and str(expected["start_ticks"]).isdigit(), + "store birth input", + ) + for mapping in expected["ports"]: + need( + set(mapping) == {"host", "host_port", "internal_port", "protocol"} + and mapping["host"] == "127.0.0.1" + and mapping["protocol"] == "tcp", + "store loopback mapping", + ) + if role == "clickhouse": + port = spec["ch"]["port"] + need( + expected["memory_limit_bytes"] == 5 * GIB + and expected["server_limit_bytes"] == 9 * GIB // 2, + "fixed CH caps", + ) + else: + target = urlsplit( + spec["pg_admin_dsn"] if role == "pg_primary" else spec["pg_standby_dsn"] + ) + need( + target.scheme in {"postgres", "postgresql"} + and target.hostname == "127.0.0.1" + and target.port + and not target.query + and not target.fragment, + "numeric owned PG endpoint", + ) + port = target.port + need( + type(expected["memory_limit_bytes"]) is int + and expected["memory_limit_bytes"] > 0 + and type(expected["shared_buffers_bytes"]) is int + and expected["shared_buffers_bytes"] > 0, + "observed PG caps", + ) + need(any(m["host_port"] == port for m in expected["ports"]), "store endpoint binding") + proof = ref(expected["owner_evidence"]) + need( + proof.get("container_id") == expected["container_id"] + and proof.get("created_at") == expected["created_at"] + and proof.get("task") == "cell-trigger-acceptance" + and proof.get("ci_run") == spec["ci_run"], + "positive creator evidence", + ) + + +def child_env(spec: dict, settings: dict, *, ca: bool = False) -> dict: + env = { + "PATH": "/usr/local/bin:/usr/bin:/bin", + "LANG": "C.UTF-8", + "TZ": "UTC", + "PYTHONPATH": spec["root"], + "LOOM_ENV_FILE": "", + "LOOM_SETTINGS_FILE": "", + "HOME": str(Path(spec["workdir"]) / "home"), + "TMPDIR": str(Path(spec["workdir"]) / "tmp"), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPYCACHEPREFIX": str(Path(spec["workdir"]) / "bytecode"), + "GITHUB_ACTIONS": os.environ.get("GITHUB_ACTIONS", ""), + "CI": os.environ.get("CI", ""), + **{ + "GITHUB_" + k: os.environ.get("GITHUB_" + k, "") + for k in ("RUN_ID", "RUN_ATTEMPT", "REPOSITORY") + }, + "LOOM_INSTANCE_LABEL": "iso:trigger:" + spec["fixture_id"][:12], + } + for key, value in settings.items(): + need(re.fullmatch(r"[a-z][a-z0-9_]*", key), "setting name") + env["LOOM_" + key.upper()] = value if isinstance(value, str) else json.dumps(value) + if ca: + env["SSL_CERT_FILE"] = str(Path(spec["workdir"]) / "ca.crt") + return env + + +def load_owned(path: Path): + from owned_children import OwnedChildren + + return OwnedChildren() + + +class Owner: + def __init__(self, spec: dict, owned, note) -> None: + self.spec, self.owned, self.note = spec, owned, note + self.leaders: list[dict] = [] + self.failures: list[list] = [] + self.phase = "owner" + self.failure_overflow = False + self.failure_attempted = 0 + self.failure_written = 0 + self.diagnostic_failures = 0 + self.interruption = Refused("interrupted") + self.stopping = False + self.terminal_reporting = False + self.blocked = False + self.started = time.monotonic() + self.serial = 0 + self.sampled = 0.0 + self.guard_stop = threading.Event() + self.guard = None + + def select(self, error: BaseException, *, phase: str | None = None) -> None: + # Select before formatting evidence. No IO in a signal handler. + if len(self.failures) >= 64: + self.failure_overflow = True + return + entry = [error, {"phase": phase or self.phase}] + self.failures.append(entry) + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + selected_phase = entry[1]["phase"] + unknown = {"file": "unknown", "line": 0, "function": "unknown"} + try: + site = unknown + frame = sys._getframe(1) + line = frame.f_lineno + trace = error.__traceback__ + for _ in range(64): + if trace is None: + break + candidate = trace.tb_frame + if Path( + candidate.f_code.co_filename + ).name in FILES and candidate.f_code.co_name not in { + "need", + "closed", + "equal", + "integer", + "identifier", + "hex64", + }: + frame = candidate + line = trace.tb_lineno + trace = trace.tb_next + while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: + frame = frame.f_back + if frame is not None: + line = frame.f_lineno + if frame is not None and Path(frame.f_code.co_filename).name in FILES: + site = { + "file": Path(frame.f_code.co_filename).name, + "line": line, + "function": frame.f_code.co_name[:64], + } + entry[1] = { + "phase": selected_phase, + "error_type": type(error).__name__[:64], + "site": site, + "outcome": "refused" + if isinstance(error, Refused) + else "error" + if isinstance(error, Exception) + else "cancelled", + } + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + entry[1] = { + "phase": selected_phase, + "error_type": type(error).__name__[:64], + "site": unknown, + "outcome": "evidence-error", + } + if len(self.failures) < 64: + self.failures.append( + [ + secondary, + { + "phase": "diagnostic", + "error_type": type(secondary).__name__[:64], + "site": unknown, + "outcome": "error", + }, + ] + ) + else: + self.failure_overflow = True + if len(self.failures) > 64: + self.failure_overflow = True + del self.failures[64:] + + def failure_summary(self) -> dict: + records = [] + snapshot = self.failures[:64] + for index, (_, record) in enumerate(snapshot): + if "outcome" not in record: + break + records.append(dict(index=index, primary=index == 0, **record)) + return { + "records": records, + "pending_records": len(snapshot) - len(records), + "overflow": self.failure_overflow or len(self.failures) > 64, + "event_attempts": self.failure_attempted, + "event_records_written": self.failure_written, + "diagnostic_write_failures": self.diagnostic_failures, + } + + def flush_failures(self) -> None: + # Snapshot once: failed evidence writes cannot recursively create more writes here. + records = self.failure_summary()["records"] + for index in range(self.failure_attempted, len(records)): + self.failure_attempted = index + 1 + try: + self.note("failure", **records[index]) + self.failure_written += 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.diagnostic_failures += 1 + self.select(error, phase="diagnostic") + + @property + def first(self) -> BaseException | None: + return self.failures[0][0] if self.failures else None + + def record(self, event: str, **fields) -> None: + try: + self.note(event, **fields) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="diagnostic") + + def interrupt(self, _signum, _frame) -> None: + if not self.stopping: + self.stopping = True + self.select(self.interruption, phase="interrupt") + if self.terminal_reporting: + # Closed ownership permits cancellation to escape the final return boundary. + first = self.first + raise self.interruption if first is None else first + + def sample(self, *, admit: bool = False) -> None: + try: + now = time.monotonic() + available = next( + int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + need(available >= POLICY["admit_mem_bytes" if admit else "stop_mem_bytes"], "RAM floor") + for path in [ + Path(self.spec["workdir"]), + *[Path(s["disk_path"]) for s in self.spec["stores"].values()], + ]: + need(shutil.disk_usage(path).free >= 16 * GIB, "disk floor") + boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() + for store in self.spec["stores"].values(): + identity = self.owned.identity(store["host_pid"]) + need( + identity is not None + and identity[1] == store["start_ticks"] + and boot == store["boot_id"] + and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + pending = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] + seen = set() + while pending: + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + before = self.owned.identity(pid) + need( + before is not None, "enumerated resource process disappeared; sample incomplete" + ) + need(self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], "process RSS") + pending.extend(self.owned.children(pid)) + after = self.owned.identity(pid) + need( + after is not None and after[:2] == before[:2], + "resource process disappeared or changed; sample incomplete", + ) + for item in tuple(self.leaders): + need(item["log"].stat().st_size <= 16 * 1024 * 1024, "child log bound") + driver_artifact = Path(self.spec["workdir"]) / "driver.jsonl" + if driver_artifact.exists(): + need(driver_artifact.stat().st_size <= 64 * 1024 * 1024, "driver artifact bound") + summary = Path(self.spec["workdir"]) / "mint-summary.stdout" + if summary.exists(): + need(summary.stat().st_size <= 4096, "mint stdout bound") + total = 0 + for arm in range(10): + path = Path(self.spec["workdir"]) / "delivery-intent" / f"arm-{arm}.jsonl" + if path.exists(): + size = path.stat().st_size + need(size <= 32 * 1024 * 1024, "intent file bound") + total += size + need(total <= 320 * 1024 * 1024, "intent aggregate bound") + self.sampled = now + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error, phase="resource") + + def start_guard(self) -> None: + def watch() -> None: + try: + while not self.guard_stop.wait(1): + self.sample() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + self.sample(admit=True) + if self.first is not None: + raise self.first + self.guard = threading.Thread(target=watch, name="trigger-resource-readonly", daemon=False) + self.guard.start() + + def monitor(self, *, admit: bool = False, cleanup: bool = False) -> None: + self.flush_failures() + if admit: + self.sample(admit=True) + if time.monotonic() - self.started >= 7200: + self.select(Refused("global wall overrun; cleanup retains exclusion")) + if not cleanup: + try: + need( + not self.stopping and time.monotonic() - self.started < 7020, + "global work deadline", + ) + need(time.monotonic() - self.sampled < 3, "resource sampler stale") + for item in self.leaders: + if item["persistent"]: + need(item["process"].poll() is None, "fixture leader ended") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.first is not None and not cleanup: + raise self.first + + def start( + self, + role: str, + argv: list[str], + env: dict, + *, + persistent: bool = False, + cleanup: bool = False, + ) -> dict: + self.monitor(cleanup=cleanup) + source(self.spec) + argv = list(argv) + executable = shutil.which(argv[0], path=env.get("PATH")) + need(executable is not None and Path(executable).is_absolute(), "known child executable") + argv[0] = executable + self.serial += 1 + label = role + "-" + str(self.serial) + log_path = Path(self.spec["workdir"]) / (label + ".log") + fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + item = None + stdout_fd = None + try: + if role == "mint-token": + stdout_fd = os.open( + Path(self.spec["workdir"]) / "mint-summary.stdout", + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + ) + process = subprocess.Popen( + argv, + cwd=self.spec["root"], + env=env, + stdout=stdout_fd if stdout_fd is not None else fd, + stderr=fd if stdout_fd is not None else subprocess.STDOUT, + start_new_session=True, + ) + # Register Popen before any observation/callback can fail. + item = { + "role": label, + "process": process, + "pidfd": None, + "identity": None, + "persistent": persistent, + "log": log_path, + "terminal": None, + "argv": list(argv), + "configuration_sha256": digest(env), + "launch_receipt": None, + "terminal_receipt": None, + } + item["signals"] = [] + self.leaders.append(item) + identity = self.owned.identity(process.pid) + need(identity and identity[0] == self.owned.pid, "direct child birth") + need(os.stat(f"/proc/{process.pid}").st_uid == os.geteuid(), "child UID") + item["identity"] = identity + item["pidfd"] = os.pidfd_open(process.pid) + after = self.owned.identity(process.pid) + need(after is not None and after[:2] == identity[:2], "stable pinned child") + item["launch_receipt"] = write( + Path(self.spec["workdir"]) / (label + "-birth.json"), + { + "pid": process.pid, + "start_ticks": identity[1], + "ppid": identity[0], + "uid": os.geteuid(), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "cwd": self.spec["root"], + "configuration_sha256": digest(env), + "utc": utc(), + }, + ) + self.record("birth", role=label, pid=process.pid, start_ticks=identity[1]) + if self.first is not None and not cleanup: + raise self.first + return item + except BaseException as error: + self.select(error) + raise + finally: + if stdout_fd is not None: + try: + os.close(stdout_fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + try: + os.close(fd) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + def poll(self, item: dict) -> int | None: + result = item["process"].poll() + if result is not None and item["terminal"] is None: + item["terminal"] = {"returncode": result, "utc": utc()} + item["persistent"] = False + try: + item["terminal_receipt"] = write( + Path(self.spec["workdir"]) / (item["role"] + "-terminal.json"), item["terminal"] + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + self.record("terminal", role=item["role"], returncode=result) + return result + + def command( + self, role: str, argv: list[str], env: dict, *, seconds: float = 120, cleanup: bool = False + ) -> dict: + item = self.start(role, argv, env, cleanup=cleanup) + end = ( + time.monotonic() + seconds + if cleanup + else min(self.started + 7020, time.monotonic() + seconds) + ) + while self.poll(item) is None: + self.monitor(cleanup=cleanup) + need(time.monotonic() < end, "serial command deadline") + time.sleep(0.05) + need(item["process"].returncode == 0, "native command refused") + if self.first is not None and not cleanup: + raise self.first + return item + + def send(self, item: dict, sig: int) -> None: + if self.poll(item) is not None: + return + before = self.owned.identity(item["process"].pid) + if ( + before is None + or before[0] != self.owned.pid + or (item["identity"] is not None and before[:2] != item["identity"][:2]) + ): + self.blocked = True + raise Refused("child identity mismatch") + # No generic reaper runs while Popen owns this direct child, so an unreaped + # Popen leader cannot have its PID reused during an interrupted birth capture. + if item["identity"] is None: + item["identity"] = before + if item["pidfd"] is None: + item["pidfd"] = os.pidfd_open(item["process"].pid) + need(os.stat(f"/proc/{item['process'].pid}").st_uid == os.geteuid(), "signal UID") + after = self.owned.identity(item["process"].pid) + need(after is not None and after[:2] == before[:2], "signal birth recheck") + try: + signal.pidfd_send_signal(item["pidfd"], sig) + item.setdefault("signals", []).append(sig) + except ProcessLookupError: + need(self.poll(item) is not None, "lost pinned child") + + def stop_one(self, item: dict, *, driver: bool = False, api: bool = False) -> None: + if self.poll(item) is not None: + self.terminal_outcome(item, driver=driver, api=api) + return + phases = ( + [(signal.SIGINT, 60), (signal.SIGTERM, 15)] + if driver + else [(signal.SIGTERM, 90 if api else 15)] + ) + for sig, seconds in phases + [(signal.SIGKILL, 5)]: + if self.poll(item) is not None: + break + if sig == signal.SIGKILL or (driver and sig == signal.SIGTERM): + self.select(Refused("cleanup escalation")) + self.send(item, sig) + end = time.monotonic() + seconds + while self.poll(item) is None and time.monotonic() < end: + self.monitor(cleanup=True) + time.sleep(0.05) + need(self.poll(item) is not None, "child not reaped") + item["process"].wait() + self.terminal_outcome(item, driver=driver, api=api) + + def terminal_outcome(self, item: dict, *, driver: bool, api: bool) -> None: + role = item["role"].split("-", 1)[0] + result = item["process"].returncode + sent = item.get("signals", []) + if driver: + ok = result in (0, 1) and not sent + elif api or role in {"router", "proxy"}: + # Uvicorn re-raises its captured TERM after graceful shutdown; the + # private proxy consumes TERM. Only an owner-requested stop qualifies. + ok = signal.SIGTERM in sent and result in ( + (0, -signal.SIGTERM) if role != "proxy" else (0,) + ) + else: + return + if not ok: + self.select(Refused("unexpected fixture terminal outcome")) + + def cleanup(self) -> None: + # Each Popen owns its status until all leaders are reaped. + self.flush_failures() + if self.owned is None: + need(not self.leaders, "uninitialized ownership with registered children") + return + for prefix in ("driver", "api", "router", "proxy", ""): + for item in self.leaders: + if item.get("cleaned") or (prefix and not item["role"].startswith(prefix)): + continue + try: + self.stop_one( + item, + driver=item["role"].startswith("driver"), + api=item["role"].startswith("api"), + ) + item["cleaned"] = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + if self.blocked or any(self.poll(item) is None for item in self.leaders): + raise Refused("owned leader cleanup unresolved; exclusion retained") + try: + self.owned.stop(None, lambda *args, **kwargs: self.record("adopted-cleanup", **kwargs)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + need(not self.owned.children(self.owned.pid), "kernel children remain") + for item in self.leaders: + if item["pidfd"] is not None: + try: + os.close(item["pidfd"]) + item["pidfd"] = None + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.select(error) + + +def runtime(spec: dict) -> int: + """The separately admitted path; never called by inert controls.""" + from native import Fixture + + validate(spec) + source(spec) + for port in spec["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as check: + check.bind(("127.0.0.1", port)) + workdir = Path(spec["workdir"]) + need(workdir.is_absolute() and not workdir.exists(), "exclusive work directory") + os.mkdir(workdir, 0o700) + for name in ("home", "tmp", "bytecode"): + (workdir / name).mkdir(mode=0o700) + events = os.open( + workdir / "events.jsonl", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) + + def note(event: str, **fields) -> None: + raw = (json.dumps(dict(utc=utc(), event=event, **fields), allow_nan=False) + "\n").encode() + need( + len(raw) <= 65536 and os.lseek(events, 0, os.SEEK_CUR) + len(raw) <= 16 * 1024 * 1024, + "event bound", + ) + need(os.write(events, raw) == len(raw), "complete event write") + os.fsync(events) + + owner = None + lock = None + fixture = None + release = False + verdict = "INVALID" + first = None + events_closed = False + lock_closed = False + + def selected() -> BaseException | None: + retained = owner.first if owner is not None else None + return retained if retained is not None else first + + try: + owner = Owner(spec, None, note) + owner.phase = "ownership" + owner.owned = load_owned(Path(spec["helper_path"])) + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, owner.interrupt) + owner.start_guard() + owner.monitor(admit=True) + lock = os.open( + "/tmp/loom-full-unit-suite.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600 + ) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + # This isolated job owns the canonical slot through final native cleanup. + fixture = Fixture(spec, owner) + owner.phase = "setup" + fixture.setup() + owner.monitor(admit=True) + owner.phase = "measure" + result = fixture.measure() + owner.phase = "provenance" + fixture.provenance() + verdict = result + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + if owner is not None: + owner.select(error) + finally: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + else: + release = True + if not release: + # Never exit over unreaped children or uncertain task writes. + while True: + if owner is not None: + try: + owner.phase = "cleanup" + owner.cleanup() + if fixture is not None: + owner.phase = "teardown" + fixture.teardown() + owner.phase = "cleanup" + owner.cleanup() + release = True + break + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + try: + time.sleep(1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error) + if owner is not None: + try: + owner.phase = "monitor-stop" + owner.monitor(cleanup=True) + owner.guard_stop.set() + if owner.guard is not None: + owner.guard.join() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error) + owner.flush_failures() + first = owner.first if owner.first is not None else first + try: + os.fsync(events) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + finally: + try: + os.close(events) + events_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="events-close") + first = error if first is None else first + first = selected() + if first is not None: + verdict = "INVALID" + try: + write( + workdir / "outer-result.json", + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "candidate_verdict": verdict, + "native_cleanup_complete": release + and (fixture is None or not fixture.teardown_incomplete), + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + "utc": utc(), + }, + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="result") + first = error if first is None else first + verdict = "INVALID" + if lock is not None and release: + try: + os.close(lock) + lock_closed = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="unlock") + first = error if first is None else first + verdict = "INVALID" + if owner is not None and release and events_closed and (lock is None or lock_closed): + try: + owner.terminal_reporting = True + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + owner.select(error, phase="reporting-mode") + try: + first = selected() + if first is not None: + verdict = "INVALID" + # Candidate bytes require successful terminal status; they grant no PASS alone. + print( + json.dumps( + { + "candidate_verdict": verdict, + "error_type": type(first).__name__ if first is not None else "", + "failure_evidence": owner.failure_summary() if owner is not None else None, + } + ), + flush=True, + ) + return 0 if verdict == "PASS" and selected() is None else 1 + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(error, phase="stdout") + first = selected() + first = error if first is None else first + verdict = "INVALID" + try: + write( + workdir / "reporting-failure.json", + { + "schema": 2, + "verdict": "INVALID", + "error_type": type(first).__name__, + "failure_evidence": owner.failure_summary() if owner is not None else None, + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + if owner is not None: + owner.select(secondary, phase="reporting") + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spec", required=True, type=Path) + args = parser.parse_args() + try: + return runtime(read(args.spec)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(json.dumps({"verdict": "INVALID", "error_type": type(error).__name__}), flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py new file mode 100644 index 0000000000..a4c3661fb5 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -0,0 +1,2391 @@ +"""Native operations for the isolated trigger acceptance fixture.""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import hashlib +import json +import os +import re +import socket +import sys +import time +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +from capture import ( + Codec, + Workload, + closed, + equal, + hex64, + identifier, + integer, + json_object, + mint_metadata, + protocol, + verify_token, +) +from fixture import ( + FILES, + GIB, + HERE, + POLICY, + PROTOCOL_SHA, + Refused, + child_env, + database, + digest, + need, + pg_dsn, + read, + ref, + sha, + utc, + write, +) + +COUNTS = ("head_global", "head_scope", "revision_global", "revision_scope") +DEBT = ( + "delivery_global", + "delivery_scope", + "audit_global", + "audit_scope", + "error_global", + "error_scope", +) +PG_FIELDS = ( + tuple( + "c_" + k + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + ) + + ("head",) + + tuple( + "r_" + k + for k in ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + ) + + ("revision", "audit", "targets", "attempts", "last_error") +) + + +def pg_census_sql(): + # One statement snapshot supplies all ten fixed counts and server observations. + counts = [] + for label, table in ( + ("head", "memory_current"), + ("revision", "memory_revisions"), + ("delivery", "memory_deliveries"), + ("audit", "audit_outbox"), + ("error", "memory_deliveries"), + ): + predicate = {"delivery": "targets<>0", "error": "(attempts<>0 OR last_error<>'')"}.get( + label, "TRUE" + ) + limit = 45501 if label in {"head", "revision"} else 1 + for scope in ("global", "scope"): + where = predicate + (" AND org=$1 AND namespace=$2" if scope == "scope" else "") + counts.append( + f"(SELECT count(*) FROM (SELECT 1 FROM loom.{table} WHERE " + f"{where} LIMIT {limit}) bounded) AS {label}_{scope}" + ) + return ( + "SELECT " + + ",".join(counts) + + ( + ",CASE WHEN octet_length(pg_current_snapshot()::text)<=16384 " + "THEN pg_current_snapshot()::text ELSE NULL END AS " + "snapshot,current_setting('transaction_isolation') AS " + "isolation,current_setting('transaction_read_only') AS " + "read_only,current_database() AS database,pg_backend_pid() " + "AS backend_pid" + ) + ) + + +def pg_record(value, fields): + need( + value is not None and list(value.keys()) == list(fields), "closed ordered PG result columns" + ) + return dict(value) + + +async def begin_observation(conn, context, isolation, *, backend=None): + initial = conn.is_in_transaction() + need(initial is False, "idle PG snapshot boundary") + statement = "BEGIN ISOLATION LEVEL " + isolation.upper() + " READ ONLY" + status = await conn.execute(statement) + active = conn.is_in_transaction() + need(status == "BEGIN" and active is True, "PG read-only BEGIN acknowledged") + value = pg_record( + await conn.fetchrow(pg_census_sql(), context["org"], context["namespace"]), + ( + *COUNTS[:2], + *COUNTS[2:], + *DEBT[:2], + *DEBT[2:4], + *DEBT[4:], + "snapshot", + "isolation", + "read_only", + "database", + "backend_pid", + ), + ) + for key in COUNTS: + need(integer(value[key]) == 45500, "exact global/scoped PG census") + for key in DEBT: + need(integer(value[key], 0, 1) == 0, "settled global/scoped PG debt") + snapshot = value.pop("snapshot") + need( + type(snapshot) is str + and len(snapshot.encode()) <= 16384 + and re.fullmatch(r"[0-9]+:[0-9]+:(?:[0-9]+(?:,[0-9]+)*)?", snapshot), + "actual bounded PG snapshot", + ) + need( + value["isolation"] == isolation + and value["read_only"] == "on" + and value["database"] == context["databases"]["pg_cell"], + "actual PG snapshot settings/database", + ) + integer(value["backend_pid"], 1, 2**31 - 1) + need(backend is None or backend == value["backend_pid"], "same owned PG backend") + return dict( + snapshot=snapshot, + snapshot_bytes=len(snapshot.encode()), + snapshot_sha256=hashlib.sha256(snapshot.encode()).hexdigest(), + isolation=value["isolation"], + read_only=value["read_only"], + database=value["database"], + backend_pid=value["backend_pid"], + begin_status=status, + initial_transaction_state=initial, + active_transaction_state=active, + counts={key: value[key] for key in (*COUNTS, *DEBT)}, + ) + + +async def end_observation(conn, observation): + status = await conn.execute("COMMIT") + ended = conn.is_in_transaction() + need(status == "COMMIT" and ended is False, "PG snapshot end acknowledged") + observation.update(end_status=status, ended_transaction_state=ended) + + +async def ch_result(ch, query, parameters, fields, limit): + result = await ch.query( + query, + parameters=parameters, + settings={ + "max_result_rows": limit, + "max_result_bytes": 20 * 1024 * 1024, + "result_overflow_mode": "throw", + "max_execution_time": 30, + }, + ) + need(list(result.column_names) == list(fields), "closed ordered CH result columns") + data = result.result_rows + need( + type(data) in {list, tuple} + and len(data) <= limit + and all(type(row) in {list, tuple} and len(row) == len(fields) for row in data), + "bounded CH tuples before maps", + ) + return data + + +async def ch_census(ch, context): + db = database(context["databases"]["ch_schema"]) + observed = {} + for label, table in (("memory", "memory"), ("audit", "audit_log")): + for scope in ("global", "scope"): + where = "event_type='memory.revision'" if label == "audit" else "1" + if scope == "scope": + where += " AND org={org:String} AND namespace={ns:String}" + result = await ch_result( + ch, + f"SELECT count() AS n FROM (SELECT 1 FROM `{db}`.{table}" + + (" FINAL" if label == "memory" else "") + + f" WHERE {where} LIMIT 45501)", + {"org": context["org"], "ns": context["namespace"]}, + ("n",), + 1, + ) + need( + len(result) == 1 and integer(result[0][0]) == 45500, + "exact CH global/scoped workload census", + ) + observed[label + "_" + scope] = result[0][0] + return observed + + +def pg_page_sql(): + c = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "seq", + "content_sha256", + ) + r = ( + "org", + "namespace", + "seq", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + "request_id", + ) + fields = ( + [f"c.{k} AS c_{k}" for k in c] + + ["c.row::text AS head"] + + [f"r.{k} AS r_{k}" for k in r] + + [ + "r.row::text AS revision", + "r.audit::text AS audit", + "d.targets", + "d.attempts", + "d.last_error", + ] + ) + text = ["c." + k for k in c if k not in {"memory_id", "version", "seq"}] + [ + "r." + k for k in r if k not in {"memory_id", "version", "seq"} + ] + sizes = " AND ".join(f"octet_length({k})<=512" for k in text) + return ( + "SELECT " + + ",".join(fields) + + ( + " FROM loom.memory_current c JOIN loom.memory_revisions r ON " + "(r.org,r.namespace,r.seq)=(c.org,c.namespace,c.seq) LEFT " + "JOIN loom.memory_deliveries d ON " + "(d.org,d.namespace,d.seq)=(c.org,c.namespace,c.seq) WHERE " + "c.org=$1 AND c.namespace=$2 AND c.memory_id=ANY($3::uuid[]) " + "AND octet_length(c.row::text)<=65536 AND " + "octet_length(r.row::text)<=65536 AND " + "octet_length(r.audit::text)<=16384 AND " + ) + + sizes + + ( + " AND (d.last_error IS NULL OR " + "octet_length(d.last_error)<=512) ORDER BY " + "c.memory_id,c.agent,c.user_id,c.session_id,c.memory_key " + "LIMIT $4" + ) + ) + + +def complete_ids(values, expected): + # Inspect multiplicity BEFORE constructing lookup maps. + need( + len(values) == len(expected) + and len(set(values)) == len(values) + and set(values) == set(expected), + "complete unique result multiset before map", + ) + + +def typed_revision(stored, row, audit, item, seen): + for prefix, names in ( + ( + "c_", + ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_id", + "version", + "content_sha256", + ), + ), + ( + "r_", + ( + "org", + "namespace", + "user_id", + "memory_id", + "version", + "content_sha256", + "actor", + "source_class", + ), + ), + ): + for key in names: + value = stored[prefix + key] + if key == "memory_id": + value = identifier(value) + elif key == "version": + value = integer(value, 1, 2**63 - 1) + else: + need(type(value) is str and len(value.encode()) <= 512, "typed PG identity text") + equal(value, row[key]) + for key, expected in ( + ("memory_key", item["key"]), + ("memory_id", item["memory_id"]), + ("version", item["version"]), + ("content_sha256", item["content_sha256"]), + ): + equal(row[key], expected) + seq = integer(stored["c_seq"], 1, 2**63 - 1) + need( + integer(stored["r_seq"], 1, 2**63 - 1) == seq and seq not in seen, + "unique exact PG revision link", + ) + seen.add(seq) + need(len(seen) <= 45500, "PG seq bound") + equal(stored["r_request_id"], audit["request_id"]) + delivery = [stored[key] for key in ("targets", "attempts", "last_error")] + if delivery != [None, None, None]: + need( + integer(delivery[0], 0, 0) == 0 + and integer(delivery[1], 0, 0) == 0 + and type(delivery[2]) is str + and delivery[2] == "", + "settled typed PG delivery", + ) + return seq + + +def chain_update(chain, index, value): + raw = json.dumps( + dict(index=index, value=value), sort_keys=True, allow_nan=False, separators=(",", ":") + ).encode() + chain.update(len(raw).to_bytes(8, "big")) + chain.update(raw) + + +async def verify_pages(conn, ch, context, workload, codec, native_hash): + db = database(context["databases"]["ch_schema"]) + seen_seq = set() + seen_hash = set() + count = 0 + expected_chain = hashlib.sha256() + observed_chain = hashlib.sha256() + authority_chain = hashlib.sha256() + raw_keys = hashlib.sha256() + for page in workload.pages(): + need(0 < len(page) <= 100, "fixed proof page") + ids = [item["memory_id"] for item in page] + complete_ids(ids, ids) + fetched = await conn.fetch( + pg_page_sql(), + context["org"], + context["namespace"], + [uuid.UUID(value) for value in ids], + len(page) + 1, + ) + need(len(fetched) == len(page), "PG page exact cardinality/overflow") + values = [pg_record(row, PG_FIELDS) for row in fetched] + complete_ids([identifier(row["c_memory_id"]) for row in values], ids) + heads = {identifier(row["c_memory_id"]): row for row in values} + checked = {} + audits = {} + hashes = [] + for item in page: + stored = heads[item["memory_id"]] + head = json_object(stored["head"]) + revision = json_object(stored["revision"]) + row = codec.row(head, origin="pg", comparison=True) + other = codec.row(revision, origin="pg", comparison=True) + equal(row, other) + audit = codec.audit(json_object(stored["audit"]), origin="pg") + native = dict(audit, ts=datetime.datetime.fromisoformat(audit["ts"])) + need(native_hash(native), "actual decoded native audit hash") + seq = typed_revision(stored, row, audit, item, seen_seq) + codec.expected( + row, + audit, + context, + api=item["kind"] == "api", + metadata=workload.metadata, + driver=workload.driver_header + or dict(host=socket.gethostname(), pid=workload.driver["pid"]), + ) + if item["kind"] == "delivery": + equal(row, codec.row(item["row"], comparison=True)) + equal(audit, item["audit"]) + need(audit["row_hash"] not in seen_hash, "unique workload audit identity") + seen_hash.add(audit["row_hash"]) + hashes.append(audit["row_hash"]) + checked[item["memory_id"]] = row + audits[audit["row_hash"]] = audit + intent = { + key: item[key] for key in ("kind", "key", "memory_id", "version", "content_sha256") + } + observed = dict( + intent, + key=row["memory_key"], + memory_id=row["memory_id"], + version=row["version"], + content_sha256=row["content_sha256"], + ) + if item["kind"] == "delivery": + intent.update(row=codec.row(item["row"], comparison=True), audit=item["audit"]) + observed.update(row=row, audit=audit) + chain_update(expected_chain, count, intent) + chain_update(observed_chain, count, observed) + chain_update(authority_chain, count, dict(row=row, audit=audit, seq=seq)) + chain_update(raw_keys, count, dict(head=sorted(head), revision=sorted(revision))) + count += 1 + fields = codec.p["memory_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.memory FINAL WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND memory_id IN " + f"{{ids:Array(UUID)}} ORDER BY " + f"memory_id,agent,user_id,session_id,memory_key LIMIT " + f"{len(page) + 1}" + ), + dict( + org=context["org"], ns=context["namespace"], ids=[uuid.UUID(value) for value in ids] + ), + fields, + len(page) + 1, + ) + complete_ids([identifier(row[fields.index("memory_id")]) for row in data], ids) + for value in data: + row = codec.row(dict(zip(fields, value, strict=True)), origin="ch", comparison=True) + equal(row, checked[row["memory_id"]]) + fields = codec.p["audit_fields"] + data = await ch_result( + ch, + "SELECT " + + ",".join("`" + key + "`" for key in fields) + + ( + f" FROM `{db}`.audit_log WHERE org={{org:String}} AND " + f"namespace={{ns:String}} AND event_type='memory.revision' " + f"AND row_hash IN {{hashes:Array(String)}} ORDER BY row_hash " + f"LIMIT {len(page) + 1}" + ), + dict(org=context["org"], ns=context["namespace"], hashes=hashes), + fields, + len(page) + 1, + ) + complete_ids([hex64(row[fields.index("row_hash")], native=True) for row in data], hashes) + for value in data: + audit = codec.audit(dict(zip(fields, value, strict=True)), origin="ch") + equal(audit, audits[audit["row_hash"]]) + need(count == len(seen_seq) == len(seen_hash) == 45500, "complete exact workload identities") + need( + expected_chain.hexdigest() == observed_chain.hexdigest(), + "streamed intended/observed proof chain", + ) + return dict( + rows=count, + api=workload.counts["api"], + delivery=workload.counts["delivery"], + expected_sha256=expected_chain.hexdigest(), + observed_sha256=observed_chain.hexdigest(), + authority_sha256=authority_chain.hexdigest(), + pg_raw_key_sets_sha256=raw_keys.hexdigest(), + ) + + +def codec_contract(): + from server.memory.store.pg_memory import DATETIME_COLS + from server.memory.store.rows import FIRST_VERSION_DEFAULTS, MEMORY_PERSISTED_COLS + + specification = protocol() + codec = Codec(specification) + equal(list(MEMORY_PERSISTED_COLS), specification["memory_fields"]) + equal(FIRST_VERSION_DEFAULTS, specification["sparse_defaults"]) + equal( + sorted(DATETIME_COLS), + sorted( + key + for key, value in specification["memory_types"].items() + if value.endswith("utc_second") + ), + ) + return codec + + +async def independent_provenance(context, output): + from server.observability.audit import row_matches_hash + + codec = codec_contract() + context = dict( + context, _api_server=read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + ) + first = None + summary = None + try: + with Workload(context, codec) as workload: + async with connections(context) as (conn, ch): + try: + page = await begin_observation(conn, context, "repeatable read") + initial_ch = await ch_census(ch, context) + checked = await verify_pages( + conn, ch, context, workload, codec, row_matches_hash + ) + await end_observation(conn, page) + final = await begin_observation( + conn, context, "read committed", backend=page["backend_pid"] + ) + await end_observation(conn, final) + need( + conn.is_in_transaction() is False, + "final CH checks outside both PG snapshots", + ) + final_ch = await ch_census(ch, context) + summary = dict( + schema=2, + fixture_id=context["fixture_id"], + verified=True, + provenance=True, + **checked, + pg_page_observation=page, + pg_final_observation=final, + ch_initial_census=initial_ch, + ch_final_census=final_ch, + inputs=workload.refs, + source=dict( + head=context["head"], + tree=context["tree"], + hashes=context["source_hashes"], + ), + driver_birth=workload.driver, + api_birth=context["_api_server"], + mint_metadata=context["mint_metadata"], + hash_and_independent_stamps=True, + utc=utc(), + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + try: + if conn.is_in_transaction(): + await conn.execute("ROLLBACK") + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + need(summary is not None, "complete proof before closure") + write(output, summary) + + +def api(context: dict, output: Path) -> None: + import uvicorn + + from bench.cell_trigger_gate import _workload + from server.core import deployment_config, instances + from server.core.provider_process import _start_ticks + from server.settings import settings + + actual = settings.model_dump(mode="json") + need( + actual["router_operator_token"] == "" + and re.fullmatch(r"lcr_[0-9a-f]{128}", actual["cell_router_token"]), + "API credential posture", + ) + candidate = dict(actual, cell_router_token="") + common = {k: v for k, v in actual.items() if k != "cell_router_token"} + folder = output.parent + artifact = write( + folder / "configuration.json", + { + "schema": 2, + "api_configuration_sha256": digest(actual), + "cli_configuration_sha256": digest(candidate), + "shared_configuration_sha256": digest(common), + "workload": _workload(settings), + "fingerprints": deployment_config.fingerprints(settings), + "credential_posture": { + "api_cell_router_token_present": True, + "cli_cell_router_token_present": False, + "api_router_operator_token_present": False, + "cli_router_operator_token_present": False, + }, + }, + ) + write(folder / "private-settings.json", actual, limit=1024 * 1024) + write( + output, + { + "schema": 2, + "server": { + "pid": os.getpid(), + "start_ticks": _start_ticks(os.getpid()), + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": os.getcwd(), + "health_instance": f"{socket.gethostname()}:{os.getpid()}", + "instance_id": instances.instance_id(), + }, + "source": {"head": context["head"], "tree": context["tree"]}, + "configuration_artifact": artifact, + }, + ) + uvicorn.run( + "server.main:app", + app_dir=context["root"], + host="127.0.0.1", + port=context["ports"]["api"], + loop="asyncio", + workers=1, + reload=False, + ) + + +@asynccontextmanager +async def connections(context: dict, *, admin: bool = False): + import asyncpg + import clickhouse_connect + + from server.core import pg + + conn = ch = None + first = None + try: + dsn = ( + context["pg_admin_dsn"] + if admin + else pg_dsn(context["pg_admin_dsn"], context["databases"]["pg_cell"]) + ) + conn = await asyncpg.connect( + dsn, + timeout=10, + command_timeout=30, + server_settings={ + "application_name": "trigger-fixture:" + context["fixture_id"], + "statement_timeout": "30000", + "idle_in_transaction_session_timeout": "30000", + }, + ) + values = {k: context["ch"][k] for k in ("host", "port", "username", "password")} + ch = await clickhouse_connect.get_async_client( + **values, + database="", + query_retries=0, + connect_timeout=10, + send_receive_timeout=30, + settings={"log_comment": "fixture.private", "async_insert": 0}, + ) + yield conn, ch + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + if ch is not None: + try: + async with asyncio.timeout(15): + await ch.close() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if conn is not None: + try: + await pg.close_owned(conn, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + if first is not None: + raise first + + +async def rows(ch, query: str, parameters: dict | None = None) -> list[dict]: + result = await ch.query(query, parameters=parameters or {}) + return list(result.named_results()) + + +async def stores(context: dict, role: str, output: Path, *, drop: bool = False) -> None: + spec = context + db = database(spec["databases"][role]) + pg_role = role.startswith("pg_") + async with connections(spec, admin=True) as (conn, ch): + if drop: + creation = read(Path(spec["workdir"]) / (role + "-created.json")) + need( + creation["database"] == db and creation["fixture_id"] == spec["fixture_id"], + "exact created database", + ) + expected_id = spec["stores"]["pg_primary" if pg_role else "clickhouse"]["container_id"] + need(creation["store_container_id"] == expected_id, "created store identity") + if pg_role: + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=$1", db + ) + == 0, + "scratch PG sessions", + ) + exists = await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ) + if exists: + need( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + == creation["database_id"], + "same created PG database", + ) + await conn.execute(f'DROP DATABASE "{db}"') + else: + # Refuse asynchronous or running writes; do not flush a global queue. + aliases = [spec["databases"]["ch_schema"], spec["databases"]["ch_telemetry"]] + need( + ( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + ) + == [{"n": 0}], + "CH async writers", + ) + live = await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + need(live == [{"n": 0}], "CH running writers") + current = await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + need( + not current or current == [{"id": creation["database_id"]}], + "same created CH database", + ) + await ch.command(f"DROP DATABASE IF EXISTS `{db}` SYNC") + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "dropped": True, + "utc": utc(), + }, + ) + return + need(await conn.fetchval("SELECT NOT pg_is_in_recovery()"), "PG primary") + need( + 160000 <= int(await conn.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 primary", + ) + need(await conn.fetchval("SHOW max_prepared_transactions") == "0", "prepared transactions") + need(await conn.fetchval("SHOW synchronous_standby_names"), "synchronous standby names") + need( + await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE " + "state='streaming' AND sync_state='sync' AND flush_lsn IS " + "NOT NULL)" + ), + "synchronous standby", + ) + need( + await conn.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_primary"]["shared_buffers_bytes"], + "primary buffers", + ) + import asyncpg + + standby = await asyncpg.connect( + spec["pg_standby_dsn"], + timeout=10, + command_timeout=10, + server_settings={ + "default_transaction_read_only": "on", + "statement_timeout": "10000", + "application_name": "trigger-fixture:" + context["fixture_id"], + }, + ) + standby_error = None + try: + need(await standby.fetchval("SELECT pg_is_in_recovery()"), "standby recovery") + need( + 160000 <= int(await standby.fetchval("SHOW server_version_num")) < 170000, + "owned PG16 standby", + ) + need( + await standby.fetchval("SELECT pg_size_bytes(current_setting('shared_buffers'))") + == spec["stores"]["pg_standby"]["shared_buffers_bytes"], + "standby buffers", + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error + try: + from server.core import pg + + await pg.close_owned(standby, end=time.monotonic() + 15) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + standby_error = error if standby_error is None else standby_error + if standby_error is not None: + raise standby_error + limits = await rows( + ch, "SELECT value FROM system.server_settings WHERE name='max_server_memory_usage'" + ) + need( + len(limits) == 1 + and int(limits[0]["value"]) + == spec["stores"]["clickhouse"]["server_limit_bytes"] + == 9 * GIB // 2, + "CH server memory cap", + ) + if pg_role: + need( + not await conn.fetchval( + "SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", db + ), + "new PG database", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "utc": utc(), + }, + ) + await conn.execute(f"CREATE DATABASE \"{db}\" ENCODING 'UTF8' TEMPLATE template0") + need( + await conn.fetchval( + "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname=$1", db + ) + == "UTF8", + "created UTF8", + ) + else: + need( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + == [{"n": 0}], + "new CH database", + ) + prefix = await rows( + ch, "SELECT value FROM system.server_settings WHERE name='keeper_map_path_prefix'" + ) + need( + len(prefix) == 1 + and prefix[0]["value"].rstrip("/") == spec["ch"]["keeper_prefix"].rstrip("/"), + "actual Keeper prefix", + ) + existing = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": spec["ch"]["keeper_prefix"]}, + ) + need( + spec["databases"]["ch_schema"] not in {r["name"] for r in existing}, + "new private Keeper root", + ) + write( + Path(spec["workdir"]) / (role + "-intent.json"), + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "database": db, + "absent": True, + "keeper_root_absent": True, + "utc": utc(), + }, + ) + await ch.command(f"CREATE DATABASE `{db}` ENGINE=Atomic") + identity = ( + str(await conn.fetchval("SELECT oid FROM pg_database WHERE datname=$1", db)) + if pg_role + else ( + await rows( + ch, + "SELECT toString(uuid) AS id FROM system.databases WHERE name={db:String}", + {"db": db}, + ) + )[0]["id"] + ) + write( + output, + { + "schema": 2, + "fixture_id": spec["fixture_id"], + "role": role, + "store_container_id": spec["stores"]["pg_primary" if pg_role else "clickhouse"][ + "container_id" + ], + "database": db, + "database_id": identity, + "created_at": utc(), + "created": True, + }, + ) + + +async def resolve_creation(context: dict, role: str, output: Path) -> None: + name = database(context["databases"][role]) + async with connections(context, admin=True) as (conn, ch): + if role.startswith("pg_"): + present = bool( + await conn.fetchval("SELECT EXISTS(SELECT FROM pg_database WHERE datname=$1)", name) + ) + else: + present = ( + await rows( + ch, + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": name}, + ) + ) != [{"n": 0}] + # Presence after an unacknowledged create is not adoption authority. + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "role": role, + "database": name, + "present": present, + "utc": utc(), + }, + ) + + +def mint_retirement(group: list[dict], birth: dict, certified) -> dict: + matching = [] + for row in group: + parts = str(row["host_proof"]).split("|") + if len(parts) == 9 and (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + birth["boot_id"], + str(birth["pid"]), + str(birth["start_ticks"]), + ): + matching.append(row) + need( + len(matching) == 1 + and matching[0]["state"] == "retired" + and certified(matching[0]["host_proof"]), + "mint native positive retirement", + ) + return { + "participant_id": str(matching[0]["participant_id"]), + "state": "retired", + "version": int(matching[0]["version"]), + "birth": birth, + } + + +async def final_retirement(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + targets = read(Path(context["workdir"]) / "final-participant-births.json")["processes"] + need(0 < len(targets) <= 16, "final participant target bound") + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "final participant census bound") + observed = [] + for target in targets: + birth = ref(target["birth"]) + terminal = ref(target["terminal"]) + need(terminal["returncode"] == target["returncode"], "final terminal receipt") + observed.append( + dict( + role=target["role"], + terminal=target["terminal"], + **mint_retirement(group, birth, has_certificate), + ) + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "retired": observed, + "verified": True, + "utc": utc(), + }, + ) + + +async def mint_proof(context: dict, output: Path) -> None: + from server.core.provider_process import has_certificate + + birth = ref(read(Path(context["workdir"]) / "mint-process.json")["launch_receipt"]) + db = database(context["databases"]["ch_schema"]) + async with connections(context) as (_, ch): + group = await rows( + ch, + ( + f"SELECT participant_id,state,version,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(len(group) <= 64, "bounded native participant census") + observed = mint_retirement(group, birth, has_certificate) + write(output, {"schema": 2, "fixture_id": context["fixture_id"], **observed, "utc": utc()}) + + +async def proof( + context: dict, + output: Path, + *, + bootstrap: bool = False, + quiescent: bool = False, + keeper: bool = False, +) -> None: + from server.cli.cell_attach_beats import widened_window_s + from server.core import cell, cell_triggers + from server.settings import settings + + db = database(context["databases"]["ch_schema"]) + org, ns = context["org"], context["namespace"] + async with connections(context, admin=quiescent or keeper) as (conn, ch): + if keeper: + original = read(Path(context["workdir"]) / "cleanup-keeper-paths.json") + prefix = context["ch"]["keeper_prefix"].rstrip("/") + for entry in original["keeper_paths"]: + need(entry["path"].startswith(prefix + "/"), "recorded Keeper prefix") + parent = prefix + absent = False + for segment in entry["path"][len(prefix) + 1 :].split("/"): + children = await rows( + ch, + "SELECT name FROM system.zookeeper WHERE path={path:String}", + {"path": parent}, + ) + if segment not in {r["name"] for r in children}: + absent = True + break + parent += "/" + segment + need(absent, "owned Keeper path remains") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "keeper_paths_absent": True, + "utc": utc(), + }, + ) + return + if quiescent: + names = [context["databases"]["pg_cell"], context["databases"]["pg_directory"]] + need( + await conn.fetchval( + "SELECT count(*) FROM pg_stat_activity WHERE datname=ANY($1::text[])", names + ) + == 0, + "all scratch PG sessions closed", + ) + need( + await conn.fetchval( + ( + "SELECT count(*) FROM pg_stat_activity WHERE " + "pid<>pg_backend_pid() AND application_name=$1" + ), + "trigger-fixture:" + context["fixture_id"], + ) + == 0, + "cut-off fixture admin sessions", + ) + aliases = [db, context["databases"]["ch_telemetry"]] + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.asynchronous_inserts WHERE " + "database IN {dbs:Array(String)}" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH queue", + ) + need( + await rows( + ch, + ( + "SELECT count() AS n FROM system.processes WHERE query_id != " + "currentQueryID() AND (current_database IN " + "{dbs:Array(String)} OR arrayExists(d -> position(query, d) " + "> 0, {dbs:Array(String)}))" + ), + {"dbs": aliases}, + ) + == [{"n": 0}], + "quiescent CH writers", + ) + keeper_paths = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": aliases}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "quiescent private Keeper path") + keeper_paths.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + need(len(keeper_paths) <= 64, "Keeper census bound") + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + "pg_sessions": 0, + "ch_writers": 0, + "keeper_paths": keeper_paths, + "utc": utc(), + }, + ) + return + authority = dict( + await conn.fetchrow("SELECT authority,deployment_id,epoch FROM loom.authority") + ) + authority["deployment_id"] = str(authority["deployment_id"]) + if bootstrap: + need( + authority["authority"] == "postgres" and authority["epoch"] > 0, + "native boot authority", + ) + pair = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(pair) == 1 and all(str(pair[0][k]) == str(v) for k, v in authority.items()), + "native boot pair", + ) + params = {"org": org, "ns": ns} + need( + await rows( + ch, + ( + f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + params, + ) + == [{"owner_email": "benchmark"}], + "native registered namespace", + ) + need( + await rows( + ch, + ( + f"SELECT subject,role FROM `{db}`.namespace_grants FINAL " + f"WHERE org={{org:String}} AND namespace={{ns:String}} AND " + f"subject='benchmark' AND revoked_at IS NULL" + ), + params, + ) + == [{"subject": "benchmark", "role": "admin"}], + "native registration grant", + ) + need( + await conn.fetchval("SELECT count(*) FROM loom.memory_current") == 0, + "no bootstrap memories", + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "namespace_registered": True, + "admin_grant": True, + "memory_rows": 0, + "utc": utc(), + }, + ) + return + first = await conn.fetchrow("SELECT is_first,cell FROM loom.cell_first") + need( + first and first["is_first"] is True and first["cell"] == context["cell"], + "first cell tuple", + ) + need(await conn.fetchval("SELECT mode FROM loom.tenancy_mode") == "enforced", "enforced") + need( + await conn.fetchval("SELECT EXISTS(SELECT FROM loom.tenancy WHERE org=$1)", org), + "held fixture org", + ) + own = await conn.fetchrow("SELECT current_fp,next_fp FROM loom.cell_self") + accepted = ( + json.loads(own["current_fp"]) + if isinstance(own["current_fp"], str) + else own["current_fp"] + ) + need( + own["next_fp"] is None + and cell.matches(cell.fingerprint(str(authority["deployment_id"])), accepted), + "accepted stable fingerprint", + ) + need(not await cell_triggers.missing(conn), "canonical triggers") + paired = await rows( + ch, f"SELECT authority,deployment_id,epoch FROM `{db}`.storage_authority FINAL" + ) + need( + len(paired) == 1 and all(str(paired[0][k]) == str(v) for k, v in authority.items()), + "paired authority", + ) + for table in ("provider_configuration", "runtime_overrides"): + need( + await rows(ch, f"SELECT count() AS n FROM `{db}`.{table} FINAL") == [{"n": 0}], + "empty saved configuration", + ) + need( + await conn.fetchval( + "SELECT (SELECT count(*) FROM loom.memory_deliveries WHERE " + "targets<>0)+(SELECT count(*) FROM loom.audit_outbox)" + ) + == 0, + "settled setup", + ) + instance = context["api_instance_id"] + from types import SimpleNamespace + + from server.core import deployment_config, instances + from server.core.ch_names import NamedClient + + named = NamedClient(ch) + adapter = SimpleNamespace(query_all=lambda sql, params=None: rows(named, sql, params)) + live = await instances.read_live(adapter, all_deployments=True) + need( + len(live) == 1 + and live[0]["instance_id"] == instance + and live[0]["serving"] == 1 + and live[0]["build_commit"] == context["head"] + and live[0]["host"] == socket.gethostname() + and not live[0]["workers_lease"], + "single serving API source", + ) + need( + all(live[0].get(k) == v for k, v in deployment_config.fingerprints(settings).items()), + "live API fingerprints", + ) + from server.core.provider_process import has_certificate + + launch = read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] + participants = await rows( + ch, + ( + f"SELECT participant_id,state,host_proof FROM " + f"`{db}`._provider_participants FINAL LIMIT 65" + ), + ) + need(0 < len(participants) <= 64, "bounded native participant evidence") + serving = 0 + for participant in participants: + participant["participant_id"] = str(participant["participant_id"]) + proof_value = participant["host_proof"] + need(has_certificate(proof_value), "native participant capability") + parts = proof_value.split("|") + own = (parts[0], parts[1], parts[4], parts[5]) == ( + socket.gethostname(), + launch["boot_id"], + str(launch["pid"]), + str(launch["start_ticks"]), + ) + if own: + need(participant["state"] == "running", "serving native participant") + serving += 1 + else: + need( + participant["state"] == "retired", + "prior native participant not positively retired", + ) + need(serving == 1, "single native serving participant") + beat = await rows( + ch, + ( + f"SELECT fingerprint,cell,tenancy_mode FROM " + f"`{db}`.cell_heartbeats WHERE instance_id={{id:String}}" + ), + {"id": instance}, + ) + need(len(beat) == 1, "API heartbeat") + value = json.loads(beat[0]["fingerprint"]) + need( + value.get("serves") is True + and value.get("marked") is True + and value.get("router") is True, + "native API router state", + ) + namespace = await rows( + ch, + ( + f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + f"org={{org:String}} AND namespace={{ns:String}}" + ), + {"org": org, "ns": ns}, + ) + need(namespace == [{"owner_email": "benchmark"}], "native namespace owner") + keeper = [] + for row in await rows( + ch, + ( + "SELECT database,name,create_table_query FROM system.tables " + "WHERE database IN {dbs:Array(String)} AND engine='KeeperMap'" + ), + {"dbs": [db, context["databases"]["ch_telemetry"]]}, + ): + match = re.search(r"KeeperMap\('([^']+)'", row["create_table_query"]) + need(match and match[1].startswith("/" + db + "/"), "private actual Keeper path") + keeper.append( + { + "database": row["database"], + "table": row["name"], + "path": context["ch"]["keeper_prefix"].rstrip("/") + match[1], + } + ) + write( + output, + { + "schema": 2, + "fixture_id": context["fixture_id"], + **authority, + "first": [True, context["cell"]], + "mode": "enforced", + "held": True, + "keeper_paths": keeper, + "native_participants": participants, + "widened_window_s": widened_window_s(), + "utc": utc(), + }, + ) + + +class Fixture: + def __init__(self, spec: dict, owner) -> None: + self.s, self.o = dict(spec), owner + self.w = Path(spec["workdir"]) + self.receipts = [] + self.created = [] + self.teardown_incomplete = False + self.retirement_attempted = False + self.retirement_receipt = None + self.settings: dict = {} + self.api = None + self.cell = "gate-" + spec["fixture_id"][:12] + self.org, self.namespace = ( + "trigger-" + spec["fixture_id"], + "trigger-" + spec["fixture_id"][:12], + ) + self.s.update(cell=self.cell, org=self.org, namespace=self.namespace) + self.context = self.w / "context.json" + write(self.context, self.s) + + def env(self, settings: dict | None = None, *, ca: bool = False) -> dict: + return child_env(self.s, self.settings if settings is None else settings, ca=ca) + + def cli(self, *args: str, settings: dict | None = None, seconds: float = 120): + return self.o.command( + args[0], + [self.s["python"], "-m", "server.cli", *args], + self.env(settings, ca=True), + seconds=seconds, + ) + + def receipt(self, operation: str, value: dict) -> None: + reference = write( + self.w / (operation + "-receipt.json"), {"schema": 2, "operation": operation, **value} + ) + self.receipts.append({"operation": operation, "receipt": reference}) + + def command_receipt(self, operation: str, item: dict, **observations) -> None: + self.receipt( + operation, + { + "returncode": item["process"].returncode, + **observations, + "process": self.process_receipt(item), + "log": { + "path": str(item["log"]), + "sha256": sha(item["log"]), + "bytes": item["log"].stat().st_size, + }, + "utc": utc(), + }, + ) + + def action( + self, task: str, *, role: str = "", settings: dict | None = None, cleanup: bool = False + ) -> dict: + target = self.w / ( + role + "-created.json" + if task == "create" + else task + ("-" + role if role else "") + ".json" + ) + if task == "quiescent": + target = self.w / ("quiescent-" + str(self.o.serial + 1) + ".json") + argv = [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + task, + "--context", + str(self.context), + "--output", + str(target), + ] + if role: + argv += ["--role", role] + try: + self.o.command( + task, + argv, + self.env(settings), + seconds=600 if task == "provenance" else 120, + cleanup=cleanup, + ) + except BaseException as error: + # A terminal cleanup refusal is recorded, never retried as another mutation. + if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: + self.o.select(error) + return {"incomplete": True, "task": task, "error_type": type(error).__name__} + raise + return read(target) + + def http( + self, path: str, *, body: dict | None = None, token: str = "", tls: bool = False + ) -> dict: + import ssl + + import httpx + + port = self.s["ports"]["tls" if tls else "api"] + verify = ssl.create_default_context(cafile=str(self.w / "ca.crt")) if tls else True + context = httpx.Client(timeout=10, follow_redirects=False, trust_env=False, verify=verify) + client = context.__enter__() + try: + response = client.request( + "POST" if body is not None else "GET", + f"{'https' if tls else 'http'}://127.0.0.1:{port}{path}", + json=body, + headers={"Authorization": "Bearer " + token} if token else {}, + ) + need(response.status_code == 200, "native HTTP refused") + value = response.json() + need(isinstance(value, dict), "native HTTP shape") + except BaseException as error: + try: + context.__exit__(type(error), error, error.__traceback__) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(error) + self.o.select(secondary) + raise + else: + context.__exit__(None, None, None) + self.o.monitor() + return value + + def wait_api(self) -> None: + import httpx + + end = min(time.monotonic() + 120, self.o.started + 7020) + while True: + self.o.monitor() + try: + value = self.http("/v1/health") + ready = value.get("readiness", {}) + checks = ready.get("checks", {}) + required = { + "draining", + "clickhouse", + "migrations", + "revocation", + "keyring", + "tracker_registry", + "runtime_overrides", + "calibration", + "config_drift", + "cell", + "sandbox", + } + if self.settings["cell"]: + required.add("cell_marker") + need( + value.get("ch_ok") is True + and value.get("pg_ok") is True + and value.get("storage_authority") == "postgres" + and value.get("pending_migrations") == [] + and ready.get("ready") is True + and ready.get("failing") == [] + and required <= set(checks) + and all( + checks[name].get("required", True) is True + and checks[name].get("ok") is True + for name in required + ) + and all( + isinstance(check, dict) + and type(check.get("ok")) is bool + and (check.get("required", True) is False or check["ok"] is True) + for check in checks.values() + ), + "healthy ready API stores", + ) + return + except (OSError, httpx.TransportError, Refused): + need(time.monotonic() < end, "API boot deadline") + time.sleep(1) + + def start_api(self, stage: str) -> None: + folder = self.w / stage + folder.mkdir(mode=0o700) + self.api = self.o.start( + "api-" + stage, + [ + self.s["python"], + str(HERE / FILES[1]), + "--task", + "api", + "--context", + str(self.context), + "--output", + str(folder / "launch.json"), + ], + self.env(), + persistent=True, + ) + self.wait_api() + self.launch = read(folder / "launch.json") + self.launch_path = folder / "launch.json" + + def setup(self) -> None: + import secrets + + # docker inspect is read-only and the input names exact positive IDs. + self.inspect_stores() + for role in ("pg_cell", "pg_directory", "ch_schema", "ch_telemetry"): + output = self.w / (role + "-created.json") + self.action("create", role=role) + # Native action writes the authoritative creation receipt, not this observation. + need(read(output)["created"] is True, "actual database creation receipt") + self.created.append(role) + cell = self.cli("cell", "credential", "new", settings={}) + operator = self.cli("cell", "credential", "new", "--operator", settings={}) + + def credential(item, prefix): + text = item["log"].read_text() + match = re.search(r"^credential: (" + prefix + r"[0-9a-f]{128})$", text, re.M) + need(match, "native credential output") + return match[1] + + cell_token = credential(cell, "lcr_") + self.operator = credential(operator, "lco_") + p, db = self.s["ports"], self.s["databases"] + self.settings = dict( + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_user=self.s["ch"]["username"], + ch_password=self.s["ch"]["password"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_cell"]), + storage_authority="postgres", + pg_durability="synchronous", + embedding_provider="fake", + embedding_model="fake-ngram-1024", + embedding_dim=1024, + llm_provider="fake", + llm_model="fake-echo-1", + require_real_providers=False, + background_workers="off", + pg_delivery_batch_rows=500, + auth_required=True, + identity_provider="gateway", + self_signup="off", + mcp_auth_mode="required", + jwt_secret=secrets.token_hex(32), + scim_token="", + cell="", + cell_probe_port=p["probe"], + cell_router_url=f"http://127.0.0.1:{p['router']}", + cell_router_token=cell_token, + router_operator_token="", + public_base_url=f"https://127.0.0.1:{p['tls']}", + build_commit=self.s["head"], + bind_host="127.0.0.1", + document_s3_bucket="", + ) + router = dict( + self.settings, + cell_router_token="", + router_pg_dsn=pg_dsn(self.s["pg_admin_dsn"], db["pg_directory"]), + router_operator_sha256=__import__("hashlib").sha256(self.operator.encode()).hexdigest(), + router_shared_host=f"127.0.0.1:{p['tls']}", + router_upstream_url=f"http://127.0.0.1:{p['api']}", + router_upstream_probe_url=f"http://127.0.0.1:{p['probe']}", + ) + self.command_receipt("directory_migrate", self.cli("router", "migrate", settings=router)) + self.o.start( + "router", + [ + self.s["python"], + "-m", + "server.cli", + "router", + "serve", + "--host", + "127.0.0.1", + "--port", + str(p["router"]), + ], + self.env(router), + persistent=True, + ) + self.certificate() + self.start_api("before-cell") + self.o.start( + "proxy", + [ + self.s["python"], + str(HERE / FILES[2]), + "--listen-port", + str(p["tls"]), + "--upstream-port", + str(p["api"]), + "--certificate", + str(self.w / "server.crt"), + "--key", + str(self.w / "server.key"), + ], + self.env({}), + persistent=True, + ) + minted = self.cli( + "mint-token", + "--subject", + "benchmark", + "--org", + self.org, + "--namespaces", + self.namespace, + "--roles", + "writer", + "--service-account", + "--ttl-days", + "1", + "--save", + str(self.w / "writer.jwt"), + "--json", + ) + write(self.w / "mint-process.json", self.process_receipt(minted)) + retirement = self.action("mint-proof") + metadata, self.token = mint_metadata(self.s, self.process_receipt(minted)) + self.s["mint_metadata"] = write(self.w / "mint-metadata.json", metadata) + self.command_receipt( + "mint", minted, retirement=retirement, metadata=self.s["mint_metadata"] + ) + result = self.http( + "/v1/namespace.register", + token=self.token, + body={"org": self.org, "namespace": self.namespace, "owner_email": "benchmark"}, + ) + need( + result.get("registered") is True and result.get("namespace") == self.namespace, + "native namespace registration", + ) + write(self.w / "namespace-register.json", result) + self.receipt("namespace_register", result) + self.receipt("authority_boot", self.action("bootstrap-proof")) + import httpx + + end = min(time.monotonic() + 30, self.o.started + 7020) + while True: + self.o.monitor() + try: + self.http("/v1/health/deployment", tls=True) + break + except httpx.TransportError: + need(time.monotonic() < end, "TLS listener startup") + time.sleep(0.1) + operator_settings = dict( + self.settings, cell=self.cell, router_operator_token=self.operator, cell_router_token="" + ) + self.command_receipt( + "prepare", + self.cli( + "cell", + "prepare", + "--cell", + self.cell, + "--host", + f"127.0.0.1:{p['tls']}", + "--credential-sha256", + __import__("hashlib").sha256(cell_token.encode()).hexdigest(), + settings=operator_settings, + seconds=600, + ), + ) + self.o.stop_one(self.api, api=True) + self.settings["cell"] = self.cell + self.start_api("serving-cell") + # Read the actual source's widened interval in a bounded metadata child. + interval = self.action("window")["seconds"] + end = time.monotonic() + interval + while time.monotonic() < end: + self.o.monitor() + time.sleep(1) + self.command_receipt( + "attach", + self.cli( + "cell", + "attach", + "--cell", + self.cell, + "--wait", + "600", + "--scan-budget", + "600", + "--ch-allow-user", + self.s["ch"]["username"], + settings=operator_settings, + seconds=650, + ), + ) + self.s["api_instance_id"] = self.launch["server"]["instance_id"] + self.context = self.w / "attached-context.json" + write(self.context, self.s) + self.cli_settings = dict(self.settings, cell_router_token="") + self.attached = self.action("proof", settings=self.cli_settings) + self.receipt("post_attach_proof", self.attached) + + def inspect_stores(self, *, cleanup: bool = False) -> None: + for expected in self.s["stores"].values(): + command = self.o.command( + "inspect", + ["docker", "inspect", expected["container_id"]], + self.env({}), + cleanup=cleanup, + ) + raw = json.loads(command["log"].read_text()) + need(len(raw) == 1, "one inspected container") + actual = raw[0] + need( + actual["Id"] == expected["container_id"] + and actual["Created"] == expected["created_at"] + and actual["State"]["Running"] is True + and actual["State"]["Pid"] == expected["host_pid"] + and actual["HostConfig"]["Memory"] == expected["memory_limit_bytes"], + "current owned container", + ) + observed = [] + for internal, bindings in actual["NetworkSettings"]["Ports"].items(): + for binding in bindings or []: + observed.append( + { + "host": binding["HostIp"], + "host_port": int(binding["HostPort"]), + "internal_port": int(internal.split("/")[0]), + "protocol": internal.split("/")[1], + } + ) + need( + sorted(observed, key=digest) == sorted(expected["ports"], key=digest), + "actual loopback bindings", + ) + need( + expected["disk_path"] in {m["Source"] for m in actual["Mounts"]}, + "owned store filesystem", + ) + identity = self.o.owned.identity(expected["host_pid"]) + need( + identity + and identity[1] == expected["start_ticks"] + and os.stat(f"/proc/{expected['host_pid']}").st_uid == expected["uid"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == expected["boot_id"], + "store process identity", + ) + + def certificate(self) -> None: + def openssl(*args): + self.o.command("openssl", ["openssl", *args], self.env({})) + + previous = os.umask(0o077) + try: + openssl( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "ca.key"), + "-out", + str(self.w / "ca.crt"), + "-days", + "1", + "-subj", + "/CN=local-trigger-ca", + ) + openssl( + "req", + "-newkey", + "rsa:2048", + "-noenc", + "-keyout", + str(self.w / "server.key"), + "-out", + str(self.w / "server.csr"), + "-subj", + "/CN=127.0.0.1", + ) + extension = self.w / "server.ext" + extension.write_text("subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n") + openssl( + "x509", + "-req", + "-in", + str(self.w / "server.csr"), + "-CA", + str(self.w / "ca.crt"), + "-CAkey", + str(self.w / "ca.key"), + "-CAcreateserial", + "-out", + str(self.w / "server.crt"), + "-days", + "1", + "-extfile", + str(extension), + ) + finally: + os.umask(previous) + openssl("verify", "-CAfile", str(self.w / "ca.crt"), str(self.w / "server.crt")) + dates = self.o.command( + "certificate-dates", + ["openssl", "x509", "-in", str(self.w / "server.crt"), "-noout", "-dates"], + self.env({}), + ) + values = dict(line.split("=", 1) for line in dates["log"].read_text().splitlines()) + parsed = { + key: datetime.datetime.strptime(value, "%b %d %H:%M:%S %Y %Z") + .replace(tzinfo=datetime.UTC) + .isoformat() + for key, value in values.items() + } + private = write( + self.w / "tls-files.json", + { + "schema": 2, + "files": [ + { + "path": str(self.w / name), + "uid": (self.w / name).stat().st_uid, + "mode": (self.w / name).stat().st_mode & 0o777, + "bytes": (self.w / name).stat().st_size, + "sha256": sha(self.w / name), + } + for name in ("ca.key", "ca.crt", "server.key", "server.crt") + ], + }, + ) + self.tls = { + "ca_certificate_sha256": sha(self.w / "ca.crt"), + "server_certificate_sha256": sha(self.w / "server.crt"), + "server_ip_san": "127.0.0.1", + "not_before": parsed["notBefore"], + "not_after": parsed["notAfter"], + "listener": f"https://127.0.0.1:{self.s['ports']['tls']}", + "upstream": f"http://127.0.0.1:{self.s['ports']['api']}", + "private_files_receipt": private, + } + + def measure(self) -> str: + # Receipt construction is explicit; no old single-digest schema is accepted. + config = ref(self.launch["configuration_artifact"]) + ownership = self.ownership() + ownership_ref = write(self.w / "ownership.json", ownership) + p, db = self.s["ports"], self.s["databases"] + fixture = dict( + schema=2, + base_url=f"http://127.0.0.1:{p['api']}", + pg_host="127.0.0.1", + pg_port=url_port(self.s["pg_admin_dsn"]), + pg_database=db["pg_cell"], + pg_internal_port=5432, + ch_host="127.0.0.1", + ch_port=self.s["ch"]["port"], + ch_database=db["ch_schema"], + ch_telemetry_database=db["ch_telemetry"], + cell=self.cell, + org=self.org, + namespace=self.namespace, + deployment_id=str(self.attached["deployment_id"]), + epoch=self.attached["epoch"], + instance_id=self.launch["server"]["instance_id"], + build_commit=self.s["head"], + source_tree=self.s["tree"], + launch_manifest_sha256=sha(self.launch_path), + ownership_receipt_sha256=ownership_ref["sha256"], + background_workers="off", + controller_excluded=True, + **{ + k: config[k] + for k in ( + "api_configuration_sha256", + "cli_configuration_sha256", + "shared_configuration_sha256", + ) + }, + ) + write(self.w / "fixture.json", fixture) + self.action("resolve", settings=self.cli_settings) + metadata = ref(self.s["mint_metadata"]) + need(verify_token(metadata) == self.token, "native bearer unchanged before driver") + item = self.o.start( + "driver", + [ + self.s["python"], + str(HERE / FILES[3]), + "--context", + str(self.context), + "--capture-directory", + str(self.w / "delivery-intent"), + "--", + "--base-url", + fixture["base_url"], + "--pg-dsn", + self.cli_settings["pg_dsn"], + "--token", + self.token, + "--org", + self.org, + "--namespace", + self.namespace, + "--fixture-manifest", + str(self.w / "fixture.json"), + "--launch-manifest", + str(self.launch_path), + "--output", + str(self.w / "driver.jsonl"), + ], + self.env(self.cli_settings), + ) + write( + self.w / "driver-birth.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + while self.o.poll(item) is None: + self.o.monitor() + time.sleep(0.05) + write( + self.w / "driver-finished.json", + dict(self.process_receipt(item), schema=2, fixture_id=self.s["fixture_id"]), + ) + need(verify_token(metadata) == self.token, "native bearer unchanged after driver") + lines = item["log"].read_bytes().splitlines() + need(lines and len(lines[-1]) <= 65536, "driver final result") + result = json.loads(lines[-1]) + need( + result.get("verdict") in {"PASS", "FAIL"} + and item["process"].returncode == (0 if result["verdict"] == "PASS" else 1), + "completed driver result", + ) + return result["verdict"] + + def process_receipt(self, item: dict) -> dict: + identity = item["identity"] + args = list(item["argv"]) + for index, value in enumerate(args[:-1]): + if value in {"--token", "--pg-dsn"}: + args[index + 1] = "" + return { + "role": item["role"], + "pid": item["process"].pid, + "start_ticks": identity[1], + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "uid": os.geteuid(), + "cwd": self.s["root"], + "executable": item["argv"][0], + "executable_sha256": sha(Path(item["argv"][0])) + if Path(item["argv"][0]).is_absolute() + else sha(Path(__import__("shutil").which(item["argv"][0]))), + "source_sha256": digest(self.s["source_hashes"]), + "configuration_sha256": item["configuration_sha256"], + "redacted_argv": args, + "launch_receipt": item["launch_receipt"], + "terminal_receipt": item["terminal_receipt"], + } + + def ownership(self) -> dict: + # All entries are observations from completed setup, never a future driver PID. + need( + len(self.o.leaders) <= 64 and len(self.attached["keeper_paths"]) <= 64, + "ownership entry bounds", + ) + return { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "source": { + "root": self.s["root"], + "head": self.s["head"], + "tree": self.s["tree"], + "source_hashes": self.s["source_hashes"], + "driver_sha256": self.s["source_hashes"]["driver"], + "launcher_sha256": self.s["source_hashes"][FILES[0]], + "proxy_sha256": self.s["source_hashes"][FILES[2]], + "owned_children_sha256": self.s["source_hashes"]["owned_children.py"], + "python_executable": self.s["python"], + "python_sha256": self.s["python_sha256"], + "lockfile_sha256": self.s["lockfile_sha256"], + "dependency_receipt": self.s["dependency_receipt"], + }, + "stores": [ + {k: v for k, v in s.items() if k != "disk_path"} for s in self.s["stores"].values() + ], + "created_databases": [ + { + "role": role, + "store_container_id": read(self.w / (role + "-created.json"))[ + "store_container_id" + ], + "database": self.s["databases"][role], + "created_at": read(self.w / (role + "-created.json"))["created_at"], + "creation_receipt": { + "path": str(self.w / (role + "-created.json")), + "sha256": sha(self.w / (role + "-created.json")), + }, + "drop_required": True, + } + for role in self.created + ], + "keeper_paths": [ + dict( + row, + store_container_id=self.s["stores"]["clickhouse"]["container_id"], + creation_receipt={ + "path": str(self.w / "ch_schema-created.json"), + "sha256": sha(self.w / "ch_schema-created.json"), + }, + drop_required=True, + ) + for row in self.attached["keeper_paths"] + ], + "processes": [self.process_receipt(item) for item in self.o.leaders], + "tls": self.tls, + "native_receipts": self.receipts, + "controller_excluded": True, + "resource_policy": POLICY, + } + + def provenance(self) -> None: + self.action("provenance", settings=self.cli_settings) + + def block_teardown(self, error: BaseException) -> None: + # Retain the mutation barrier even if the negative receipt cannot be created. + self.teardown_incomplete = True + self.o.select(error) + try: + if not (self.w / "cleanup-incomplete.json").exists(): + write( + self.w / "cleanup-incomplete.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "reason": "required cleanup evidence incomplete; stores retained", + "utc": utc(), + }, + ) + except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.o.select(secondary) + + def retirement_evidence(self, retired: dict, final: list[dict]) -> None: + closed(retired, "schema fixture_id retired verified utc") + need( + type(retired["schema"]) is int + and retired["schema"] == 2 + and retired["fixture_id"] == self.s["fixture_id"] + and retired["verified"] is True + and type(retired["retired"]) is list + and len(retired["retired"]) == len(final), + "acknowledged complete final retirement", + ) + for value, item in zip(retired["retired"], final, strict=True): + closed(value, "role terminal participant_id state version birth") + equal(value["role"], item["role"]) + equal(value["terminal"], item["terminal_receipt"]) + equal(value["birth"], ref(item["launch_receipt"])) + terminal = ref(item["terminal_receipt"]) + need( + type(terminal["returncode"]) is int + and terminal["returncode"] == item["process"].returncode, + "final retirement terminal binding", + ) + need(value["state"] == "retired", "positive native retirement state") + identifier(value["participant_id"]) + integer(value["version"], 1, 2**63 - 1) + + def teardown(self) -> None: + # Only creation receipts authorize deletion, even after partial setup. + if not any( + (self.w / (role + suffix)).exists() + for role in self.s["databases"] + for suffix in ("-intent.json", "-created.json") + ): + return + self.inspect_stores(cleanup=True) + for port in self.s["ports"].values(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(1) + need(probe.connect_ex(("127.0.0.1", port)) != 0, "fixture port remains open") + quiescent = self.action("quiescent", settings={}, cleanup=True) + need( + quiescent.get("ch_writers") == quiescent.get("pg_sessions") == 0, + "server writer settlement unknown", + ) + if self.teardown_incomplete or (self.w / "cleanup-incomplete.json").exists(): + self.teardown_incomplete = True + self.o.select(Refused("known incomplete teardown; no mutation retry")) + return + if not (self.w / "cleanup-keeper-paths.json").exists(): + write(self.w / "cleanup-keeper-paths.json", {"keeper_paths": quiescent["keeper_paths"]}) + final = [item for item in self.o.leaders if item["role"].startswith(("api-", "driver-"))] + if final: + try: + if self.retirement_receipt is None: + need( + not self.retirement_attempted + and not (self.w / "final-retirement.json").exists(), + "unacknowledged final retirement artifact; no retry", + ) + self.retirement_attempted = True + need( + all( + item["terminal_receipt"] is not None + and item["launch_receipt"] is not None + for item in final + ), + "final native process receipts", + ) + write( + self.w / "final-participant-births.json", + { + "processes": [ + { + "role": item["role"], + "birth": item["launch_receipt"], + "terminal": item["terminal_receipt"], + "returncode": item["process"].returncode, + } + for item in final + ] + }, + ) + retired = self.action("final-retirement", settings={}, cleanup=True) + self.retirement_evidence(retired, final) + receipt = { + "path": str(self.w / "final-retirement.json"), + "sha256": sha(self.w / "final-retirement.json"), + } + equal(ref(receipt), retired) + self.retirement_receipt = receipt + self.retirement_evidence(ref(self.retirement_receipt), final) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.block_teardown(error) + return + for role in self.s["databases"]: + if (self.w / (role + "-intent.json")).exists() and not ( + self.w / (role + "-created.json") + ).exists(): + state = self.action("resolve-creation", role=role, settings={}, cleanup=True) + if state.get("present") is not False: + self.block_teardown(Refused("uncertain database ownership; leave untouched")) + return + for role in ("ch_telemetry", "ch_schema", "pg_directory", "pg_cell"): + created = self.w / (role + "-created.json") + if created.exists() and not (self.w / ("drop-" + role + ".json")).exists(): + result = self.action("drop", role=role, settings={}, cleanup=True) + if result.get("dropped") is not True: + self.block_teardown(Refused("native scratch cleanup incomplete")) + # A failed DROP may still run on the server; prove settlement afresh. + settled = self.action("quiescent", settings={}, cleanup=True) + need( + settled.get("ch_writers") == settled.get("pg_sessions") == 0, + "drop outcome still running", + ) + return + if not (self.w / "keeper-proof.json").exists(): + result = self.action("keeper-proof", settings={}, cleanup=True) + if result.get("keeper_paths_absent") is not True: + self.block_teardown(Refused("owned Keeper cleanup incomplete")) + return + if not (self.w / "teardown.json").exists(): + write( + self.w / "teardown.json", + { + "schema": 2, + "fixture_id": self.s["fixture_id"], + "dropped": [ + role + for role in self.s["databases"] + if (self.w / ("drop-" + role + ".json")).exists() + ], + "owned_ports_closed": True, + "pg_sessions_closed": True, + "job_container_cleanup_required": True, + "utc": utc(), + }, + ) + + +def url_port(dsn: str) -> int: + from urllib.parse import urlsplit + + port = urlsplit(dsn).port + need(port is not None, "explicit PG port") + return port + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--task", + required=True, + choices=[ + "api", + "create", + "drop", + "resolve-creation", + "mint-proof", + "final-retirement", + "proof", + "provenance", + "bootstrap-proof", + "quiescent", + "keeper-proof", + "window", + "resolve", + ], + ) + parser.add_argument("--context", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--role", default="") + args = parser.parse_args() + try: + context = read(args.context) + sys.path.insert(0, context["root"]) + if args.task == "api": + api(context, args.output) + elif args.task in {"create", "drop"}: + need(args.role in context["databases"], "fixed database role") + asyncio.run(stores(context, args.role, args.output, drop=args.task == "drop")) + elif args.task == "resolve-creation": + need(args.role in context["databases"], "fixed database role") + asyncio.run(resolve_creation(context, args.role, args.output)) + elif args.task == "window": + from server.cli.cell_attach_beats import widened_window_s + + write(args.output, {"seconds": widened_window_s()}) + elif args.task == "mint-proof": + asyncio.run(mint_proof(context, args.output)) + elif args.task == "provenance": + asyncio.run(independent_provenance(context, args.output)) + elif args.task == "final-retirement": + asyncio.run(final_retirement(context, args.output)) + elif args.task == "resolve": + from server.settings import settings + + codec_contract() + config = ref( + read(args.output.parent / "serving-cell/launch.json")["configuration_artifact"] + ) + actual = settings.model_dump(mode="json") + need( + actual["cell_router_token"] == actual["router_operator_token"] == "", + "CLI credentials", + ) + need( + digest(actual) == config["cli_configuration_sha256"] + and digest({k: v for k, v in actual.items() if k != "cell_router_token"}) + == config["shared_configuration_sha256"], + "actual CLI configuration", + ) + write( + args.output, + { + "schema": 2, + "cli_configuration_sha256": digest(actual), + "codec_protocol_sha256": PROTOCOL_SHA, + "verified": True, + }, + ) + else: + asyncio.run( + proof( + context, + args.output, + bootstrap=args.task == "bootstrap-proof", + quiescent=args.task == "quiescent", + keeper=args.task == "keeper-proof", + ) + ) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py new file mode 100644 index 0000000000..b0c5d950f8 --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -0,0 +1,145 @@ +"""Reap only exact direct and adopted children of the fixture subreaper.""" + +from __future__ import annotations + +import ctypes +import os +import signal +import subprocess +import time +from pathlib import Path + + +class OwnedChildren: + """Signal only unreaped kernel children of this private subreaper.""" + + def __init__(self) -> None: + if not hasattr(os, "pidfd_open") or not hasattr(signal, "pidfd_send_signal"): + raise RuntimeError("exact owned-process signaling needs Linux pidfds") + if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0: + error = ctypes.get_errno() + raise OSError(error, "PR_SET_CHILD_SUBREAPER: " + os.strerror(error)) + self.pid = os.getpid() + + @staticmethod + def identity(pid: int) -> tuple[int, str, str] | None: + try: + fields = Path("/proc", str(pid), "stat").read_text().rsplit(")", 1)[1].split() + return int(fields[1]), fields[19], fields[0] + except (FileNotFoundError, ProcessLookupError): + return None + + @staticmethod + def children(pid: int) -> list[int]: + return [ + int(value) + for value in Path("/proc", str(pid), "task", str(pid), "children").read_text().split() + ] + + @staticmethod + def rss(pid: int) -> int: + try: + status = Path("/proc", str(pid), "status").read_text() + except FileNotFoundError: + return 0 # Vanished sample, never proof that the process used no memory. + return next( + (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 + ) + + def stop(self, process: subprocess.Popen | None, note: object) -> None: + started = None + last_note = None + diagnostic_error = None + # Keep each verified pidfd pinned, including across numeric PID reuse. + pinned: dict[int, tuple[int, set[int]]] = {} + + def cleanup_note(**details: object) -> None: + nonlocal diagnostic_error + try: + note("cleanup-wait", **details) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + diagnostic_error = error + + try: + while True: + try: + children = self.children(self.pid) + for pid in children: + row = self.identity(pid) + if row and row[0] == self.pid and row[2] == "Z": + if ( + process is not None + and process.returncode is None + and pid == process.pid + ): + process.poll() # Preserve the real leader exit status. + else: + os.waitpid(pid, os.WNOHANG) + children = self.children(self.pid) + if not children: + break # Kernel list is empty AFTER zombie reaping. + if started is None: + started = last_note = time.monotonic() + elapsed = time.monotonic() - started + sig = ( + signal.SIGINT + if elapsed < 30 + else (signal.SIGTERM if elapsed < 40 else signal.SIGKILL) + ) + for pid in children: + row = self.identity(pid) + if row is None or row[0] != self.pid or row[2] == "Z": + continue + if pid in pinned: + descriptor, sent = pinned[pid] + try: + signal.pidfd_send_signal(descriptor, 0) + except ProcessLookupError: + os.close(descriptor) + del pinned[pid] + if pid not in pinned: + try: + descriptor = os.pidfd_open(pid) + except ProcessLookupError: + continue + try: + exact = self.identity(pid) + if exact is None or exact[0] != self.pid or exact[1] != row[1]: + os.close(descriptor) + continue + except BaseException: + os.close(descriptor) + raise + pinned[pid] = (descriptor, set()) + descriptor, sent = pinned[pid] + if int(sig) not in sent: + try: + signal.pidfd_send_signal(descriptor, sig) + sent.add(int(sig)) + except ProcessLookupError: + pass + if time.monotonic() - last_note >= 30: + cleanup_note(owned_direct_pids=children) + last_note = time.monotonic() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Unknown census/signal state retains exclusion until the kernel + # proves that every direct/adopted child has been reaped. + if diagnostic_error is None: + cleanup_note( + reason="owned-child cleanup failed", error=type(error).__name__ + ) + diagnostic_error = error + try: + time.sleep(0.1) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Even a sleep/clock failure cannot escape over an unreaped child. + diagnostic_error = error + if process is not None: + process.wait() + finally: + for descriptor, _sent in pinned.values(): + os.close(descriptor) + if diagnostic_error is not None: + raise RuntimeError( + "cleanup failed; kernel children are now empty and reaped" + ) from diagnostic_error diff --git a/scripts/gates/testsupport/cell_trigger/protocol.json b/scripts/gates/testsupport/cell_trigger/protocol.json new file mode 100644 index 0000000000..3d3670f64d --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/protocol.json @@ -0,0 +1,235 @@ +{ + "schema": 1, + "status": "r5 design only; no runtime or entry implementation", + "memory_fields": [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "memory_type", + "value", + "value_json", + "confidence", + "source_trace", + "version", + "tombstoned_at", + "created_at", + "expires_at", + "pinned", + "run_id", + "entities", + "memory_id", + "redacted_at", + "redaction_reason", + "status", + "conflict_with", + "duplicate_of", + "evidence_count", + "recall_count", + "last_recalled_at", + "cross_promoted_from_memory_id", + "cross_promoted_from_namespace", + "tags", + "superseded_by", + "rejection_reason", + "temporal_anchor", + "temporal_confidence", + "updated_at", + "valid_at", + "valid_to", + "invalid_at", + "char_limit", + "reinforcement_count", + "last_reinforced_at", + "mentioned_at", + "source_resource_id", + "content_sha256", + "files", + "concepts", + "verified_by", + "verified_at", + "verified_sha256", + "grounding", + "evidence", + "sensitivity", + "actor", + "source_class", + "lock_mode", + "lock_reason", + "lock_until", + "locked_by", + "locked_at", + "on_expire", + "expire_reason" + ], + "memory_types": { + "org": "string", + "namespace": "string", + "agent": "string", + "user_id": "string", + "session_id": "string", + "memory_key": "string", + "memory_type": "string", + "value": "string", + "source_trace": "string", + "run_id": "string", + "redaction_reason": "string", + "status": "string", + "duplicate_of": "string", + "cross_promoted_from_namespace": "string", + "superseded_by": "string", + "rejection_reason": "string", + "source_resource_id": "string", + "verified_by": "string", + "verified_sha256": "string", + "grounding": "string", + "evidence": "string", + "sensitivity": "string", + "actor": "string", + "source_class": "string", + "lock_mode": "string", + "lock_reason": "string", + "locked_by": "string", + "on_expire": "string", + "expire_reason": "string", + "tombstoned_at": "nullable_utc_second", + "created_at": "utc_second", + "expires_at": "nullable_utc_second", + "redacted_at": "nullable_utc_second", + "last_recalled_at": "nullable_utc_second", + "temporal_anchor": "nullable_utc_second", + "updated_at": "utc_second", + "valid_at": "nullable_utc_second", + "valid_to": "nullable_utc_second", + "invalid_at": "nullable_utc_second", + "last_reinforced_at": "utc_second", + "mentioned_at": "nullable_utc_second", + "verified_at": "nullable_utc_second", + "lock_until": "nullable_utc_second", + "locked_at": "nullable_utc_second", + "entities": "array_string", + "tags": "array_string", + "files": "array_string", + "concepts": "array_string", + "evidence_count": "uint32_not_bool", + "recall_count": "uint32_not_bool", + "reinforcement_count": "uint32_not_bool", + "confidence": "finite_exact_float32_not_bool", + "temporal_confidence": "finite_exact_float32_not_bool", + "value_json": "json_object_field", + "memory_id": "uuid", + "cross_promoted_from_memory_id": "nullable_uuid", + "conflict_with": "array_uuid", + "version": "positive_int63_not_bool", + "pinned": "uint8_0_or_1_not_bool", + "char_limit": "nullable_uint32_not_bool", + "content_sha256": "hex64_fixedstring" + }, + "sparse_defaults": { + "status": "active", + "conflict_with": [], + "duplicate_of": "", + "superseded_by": "", + "rejection_reason": "", + "invalid_at": null, + "redacted_at": null, + "redaction_reason": "", + "grounding": "", + "evidence": "", + "temporal_anchor": null, + "temporal_confidence": 0.0, + "source_resource_id": "", + "verified_by": "", + "verified_at": null, + "verified_sha256": "", + "cross_promoted_from_memory_id": null, + "cross_promoted_from_namespace": "", + "evidence_count": 1, + "recall_count": 0, + "last_recalled_at": null, + "reinforcement_count": 1, + "mentioned_at": null, + "source_trace": "", + "tombstoned_at": null, + "expires_at": null, + "pinned": 0, + "run_id": "-", + "entities": [], + "tags": [], + "files": [], + "concepts": [], + "valid_at": null, + "valid_to": null, + "char_limit": null, + "sensitivity": "", + "lock_mode": "", + "lock_reason": "", + "lock_until": null, + "locked_by": "", + "locked_at": null, + "on_expire": "", + "expire_reason": "" + }, + "audit_fields": [ + "ts", + "event_type", + "org", + "namespace", + "actor", + "subject", + "request_id", + "row_hash", + "build_commit", + "instance", + "instance_label", + "consumer", + "token_id" + ], + "audit_types": { + "ts": "utc_millisecond", + "event_type": "string", + "org": "string", + "namespace": "string", + "actor": "string", + "subject": "raw_subject_string", + "request_id": "string", + "row_hash": "hex64_fixedstring", + "build_commit": "string", + "instance": "string", + "instance_label": "string", + "consumer": "string", + "token_id": "string" + }, + "hash_byte_fields": { + "memory": [ + "content_sha256" + ], + "audit": [ + "row_hash" + ] + }, + "mint_summary_fields": [ + "jti", + "subject", + "org", + "namespaces", + "roles", + "ttl_days", + "is_service_account", + "read_scope", + "read_user_id", + "traffic_class", + "consumer", + "token_path" + ], + "page_rows": 100, + "query_limit_extra_rows": 1, + "memory_row_bytes": 65536, + "audit_row_bytes": 16384, + "query_result_bytes": 20971520, + "total_expected_rows": 45500, + "api_expected_rows": 20500, + "delivery_expected_rows": 25000 +} diff --git a/scripts/gates/testsupport/cell_trigger/tls_proxy.py b/scripts/gates/testsupport/cell_trigger/tls_proxy.py new file mode 100644 index 0000000000..e579fa688d --- /dev/null +++ b/scripts/gates/testsupport/cell_trigger/tls_proxy.py @@ -0,0 +1,168 @@ +"""Private fixed-target TLS bridge; run only after separate runtime admission.""" + +from __future__ import annotations + +import argparse +import asyncio +import signal +import ssl +import sys + + +class Bridge: + def __init__(self, upstream_port: int) -> None: + self.port = upstream_port + self.tasks: set[asyncio.Task] = set() + self.stopping = asyncio.Event() + self.failure: BaseException | None = None + + def fail(self, error: BaseException) -> None: + if self.failure is None: + self.failure = error + self.stopping.set() + + async def copy(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + while data := await asyncio.wait_for(reader.read(65536), 30): + writer.write(data) + await asyncio.wait_for(writer.drain(), 30) + + async def finish(self, pumps: list[asyncio.Task], endpoints: tuple) -> BaseException | None: + first = None + for pump in pumps: + if not pump.done(): + pump.cancel() + try: + outcomes = await asyncio.gather(*pumps, return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException) and not isinstance( + outcome, asyncio.CancelledError + ): + first = outcome if first is None else first + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + for endpoint in endpoints: + if endpoint is None: + continue + try: + endpoint.close() + await asyncio.wait_for(endpoint.wait_closed(), 5) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + return first + + async def drain(self, pumps: list[asyncio.Task], endpoints: tuple, first=None): + closing = asyncio.create_task(self.finish(pumps, endpoints)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error if first is None else first + try: + outcome = closing.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + outcome = error + return outcome if first is None else first + + async def connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + if len(self.tasks) >= 8 or self.stopping.is_set(): + error = await self.drain([], (writer,)) + if error is not None: + self.fail(error) + return + self.tasks.add(task) + upstream = None + pumps: list[asyncio.Task] = [] + first = None + try: + async with asyncio.timeout(120): + source, upstream = await asyncio.wait_for( + asyncio.open_connection("127.0.0.1", self.port, limit=65536), 5 + ) + pumps = [ + asyncio.create_task(self.copy(reader, upstream)), + asyncio.create_task(self.copy(source, writer)), + ] + done, _ = await asyncio.wait(pumps, return_when=asyncio.FIRST_COMPLETED) + for pump in done: + pump.result() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + finally: + first = await self.drain(pumps, (upstream, writer), first) + self.tasks.discard(task) + if first is not None and not ( + self.stopping.is_set() and isinstance(first, asyncio.CancelledError) + ): + self.fail(first) + + async def serve(self, port: int, certificate: str, key: str) -> None: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.set_alpn_protocols(["http/1.1"]) + context.load_cert_chain(certificate, key) + loop = asyncio.get_running_loop() + for signum in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(signum, self.stopping.set) + server = await asyncio.start_server( + self.connection, + "127.0.0.1", + port, + ssl=context, + limit=65536, + ssl_handshake_timeout=5, + ssl_shutdown_timeout=5, + ) + try: + await self.stopping.wait() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + finally: + closing = asyncio.create_task(self.shutdown(server)) + while not closing.done(): + try: + await asyncio.shield(closing) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + self.fail(error) + error = closing.result() + if error is not None: + self.fail(error) + if self.failure is not None: + raise self.failure + + async def shutdown(self, server) -> BaseException | None: + first = None + try: + server.close() + await server.wait_closed() + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + first = error + for task in tuple(self.tasks): + task.cancel() + result = await self.finish(list(self.tasks), ()) + return result if first is None else first + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen-port", required=True, type=int) + parser.add_argument("--upstream-port", required=True, type=int) + parser.add_argument("--certificate", required=True) + parser.add_argument("--key", required=True) + args = parser.parse_args() + if any(not 0 < port < 65536 for port in (args.listen_port, args.upstream_port)): + return 2 + if args.listen_port == args.upstream_port: + return 2 + try: + asyncio.run(Bridge(args.upstream_port).serve(args.listen_port, args.certificate, args.key)) + except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + # Error text may contain request bytes or private file names. + print(type(error).__name__, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py new file mode 100644 index 0000000000..3650641cd8 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -0,0 +1,165 @@ +"""A failed fixture may remove only the containers its creator recorded.""" + +import signal +from copy import deepcopy +from pathlib import Path + +import pytest + +from scripts.gates.cell_trigger_acceptance import Stores, native_cleanup_complete, write + + +class FakeStores(Stores): + def __init__(self, folder: Path, rows: dict): + super().__init__(folder, folder, "fixture", {"run_id": "123"}) + self.rows = deepcopy(rows) + self.removed = [] + + def inspect(self, identity, *, cleanup=False): + return self.rows[identity] + + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup + if argv[:3] == ("docker", "rm", "--force"): + self.removed.append(argv[-1]) + return "" + assert argv[:3] == ("docker", "ps", "-a") + identity = next(part.removeprefix("id=") for part in argv if part.startswith("id=")) + return identity if identity in self.rows and identity not in self.removed else "" + + +def container(identity="own", created="birth", label="fixture"): + return { + "Id": identity, + "Created": created, + "Name": "/intent", + "Config": {"Labels": {"loom.ci.trigger": label}}, + } + + +@pytest.mark.parametrize( + "change", + [ + {"Id": "reused"}, + {"Created": "new-birth"}, + {"Config": {"Labels": {"loom.ci.trigger": "foreign"}}}, + ], +) +def test_cleanup_refuses_changed_container_identity(tmp_path, change): + actual = {**container(), **change} + stores = FakeStores(tmp_path, {"own": actual}) + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + assert stores.cleanup() + assert stores.removed == [] + + +def test_cleanup_continues_exact_owned_containers_after_a_refusal(tmp_path): + stores = FakeStores(tmp_path, {"own": container(), "foreign": container(label="other")}) + stores.items = [ + {"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}, + {"role": "pg-standby", "name": "other", "id": "foreign", "created": "birth"}, + ] + assert stores.cleanup() == ["RuntimeError:pg-standby"] + assert stores.removed == ["own"] + + +def test_partial_create_requires_its_exact_name_and_random_label(tmp_path): + stores = FakeStores(tmp_path, {"intent": container()}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == [] + assert stores.removed == ["own"] + + +def test_partial_create_never_adopts_a_foreign_matching_name(tmp_path): + stores = FakeStores(tmp_path, {"intent": container(label="other")}) + stores.items = [{"role": "clickhouse", "name": "intent", "id": None, "created": None}] + assert stores.cleanup() == ["RuntimeError:clickhouse"] + assert stores.removed == [] + + +def test_cancellation_is_deferred_until_owned_cleanup_finishes(tmp_path): + stores = FakeStores(tmp_path, {}) + stores.interrupted(signal.SIGTERM, None) + assert stores.cancelled == signal.SIGTERM + stores.terminal = True + with pytest.raises(SystemExit) as caught: + stores.interrupted(signal.SIGINT, None) + assert caught.value.code == 128 + signal.SIGTERM + + +def test_store_removal_requires_acknowledged_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + with pytest.raises(FileNotFoundError): + native_cleanup_complete(tmp_path, "fixture") + write( + tmp_path / "fixture/outer-result.json", + {"schema": 2, "fixture_id": "fixture", "native_cleanup_complete": False}, + ) + with pytest.raises(RuntimeError, match="Native retirement"): + native_cleanup_complete(tmp_path, "fixture") + + +def test_failed_measurement_can_close_only_after_real_native_cleanup(tmp_path): + (tmp_path / "fixture").mkdir() + write( + tmp_path / "fixture/outer-result.json", + { + "schema": 2, + "fixture_id": "fixture", + "candidate_verdict": "FAIL", + "native_cleanup_complete": True, + }, + ) + native_cleanup_complete(tmp_path, "fixture") + + +def test_dependency_metadata_accepts_only_the_actual_python_version(tmp_path): + import hashlib + import json + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + environment = tmp_path / ".venv" + for name in fixture.CODEC_FILES: + path = environment / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("fixture codec bytes") + (environment / fixture.CODEC_FILES[1]).write_text('version = "1.9.0"\n') + (environment / fixture.CODEC_FILES[4]).write_text('__version__ = "0.32.0"\n') + spec = { + "root": str(tmp_path), + "python": str(environment / "bin/python"), + "python_sha256": "0" * 64, + "lockfile_sha256": "1" * 64, + } + + def check(version): + (environment / "pyvenv.cfg").write_text( + f"implementation = CPython\nversion_info = {version}\n" + "include-system-site-packages = false\n" + ) + receipt = { + "schema": 2, + **spec, + "versions": fixture.CODEC_VERSIONS, + "codec_sources": { + name: {"path": str(environment / name), "sha256": fixture.sha(environment / name)} + for name in fixture.CODEC_FILES + }, + } + receipt.pop("dependency_receipt", None) + path = tmp_path / "receipt.json" + path.write_text(json.dumps(receipt)) + path.chmod(0o600) + spec["dependency_receipt"] = { + "path": str(path), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + fixture.dependencies(spec, verify=True) + + check(".".join(map(str, sys.version_info[:3]))) + check("3.12") + for changed in ("3.13.0", "3.12.invalid", "3.12.999"): + with pytest.raises(fixture.Refused, match="fixed isolated CPython"): + check(changed) From a881cc97d0c7a9901af9f5a958ed5348093cf5eb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 21:47:33 +0000 Subject: [PATCH 122/181] SCIM fixtures clean only the identities they create Remove unused bot cleanup state. The person fixtures create SCIM users and org-scoped rows, while machine identities have their own owners. Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 8af632508f..0ce6b29d70 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -25,7 +25,6 @@ def __init__(self, client: ch.CHClient): self.org = unique_org("scim-person") self.run = uuid.uuid4().hex[:10] self.emails: list[str] = [] - self.bots: list[str] = [] self.results: list[tuple[str, dict[str, Any], Any]] = [] def email(self, label: str) -> str: @@ -146,12 +145,6 @@ async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) "SETTINGS mutations_sync = 2", params={"ids": [scim.user_id(e) for e in made.emails]}, ) - if made.bots: - await client.execute( - "ALTER TABLE loom.machine_identities DELETE WHERE name IN {names:Array(String)} " - "SETTINGS mutations_sync = 2", - params={"names": made.bots}, - ) await client.disconnect() From 1e71631ac673e087b011ad8ecee75bbd6e0cd84d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:06:40 +0000 Subject: [PATCH 123/181] Trigger acceptance waits for TCP and reports owned startup state Read the fixed public protocol with its own file constraints, preserve private receipt checks, select dependency inputs in CI and regenerate operation reachability. Signed-off-by: zlareb1 --- .github/workflows/docker-smoke.yml | 6 + scripts/gates/cell_trigger_acceptance.py | 131 +++++++++++++- .../gates/testsupport/cell_trigger/capture.py | 18 +- server/operations/reach_manifest.py | 1 + tests/unit/test_cell_trigger_ci_creator.py | 163 ++++++++++++++++++ 5 files changed, 310 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml index 05f3564698..8a0a64defa 100644 --- a/.github/workflows/docker-smoke.yml +++ b/.github/workflows/docker-smoke.yml @@ -37,6 +37,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] pull_request: @@ -74,6 +77,9 @@ on: "tests/unit/test_scip_checkout_fetch_ref.py", "install.sh", "pyproject.toml", + "uv.lock", + "dev/clickhouse-ci/log_queries_readonly.xml", + "dev/clickhouse-ci/custom_settings_prefixes.xml", ".github/workflows/docker-smoke.yml", ] # Weekly, so a critical CVE fixed upstream fails here before a release's scan. diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 97eedfeeab..587c0b5c4f 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -144,6 +144,7 @@ def __init__(self, root: Path, folder: Path, fixture_id: str, run: dict) -> None self.prefix = "loom-trigger-" + fixture_id[:12] self.cancelled = 0 self.terminal = False + self.phase = "store_creation" self.creation_deadline = time.monotonic() + 900 self.env = { "PATH": os.environ["PATH"], @@ -229,7 +230,11 @@ def create(self, role: str, args: list[str]) -> dict: self.command("docker", "start", identity) return intent - def wait(self, probe, seconds: int = 120) -> None: + def wait(self, phase: str, probe, seconds: int = 120) -> None: + require( + phase in {"primary_tcp", "standby_sync", "clickhouse_query"}, "Known readiness phase" + ) + self.phase = phase end = time.monotonic() + seconds while True: require(not self.cancelled, "Owned store readiness cancelled") @@ -241,6 +246,65 @@ def wait(self, probe, seconds: int = 120) -> None: require(time.monotonic() < end, "Owned store readiness deadline") time.sleep(1) + def failure_diagnostics(self) -> dict: + observations = [] + statuses = {"created", "running", "paused", "restarting", "removing", "exited", "dead"} + for item in self.items: + row = {"role": item["role"], "available": False} + observations.append(row) + if item["id"] is None: + continue + try: + records = json.loads( + self.command("docker", "inspect", item["id"], timeout=5, cleanup=True) + ) + require( + len(records) == 1 and container_matches(records[0], item, self.fixture_id), + "Diagnostic container ownership changed", + ) + state = records[0]["State"] + row.update( + available=True, + status=state["Status"] if state["Status"] in statuses else "unknown", + exit_code=state["ExitCode"] if type(state["ExitCode"]) is int else None, + oom_killed=state["OOMKilled"] if type(state["OOMKilled"]) is bool else None, + ) + if item["role"] in {"pg-primary", "pg-standby"} and state["Running"] is True: + value = self.command( + "docker", + "exec", + item["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + "SELECT CASE WHEN pg_is_in_recovery() THEN 'standby' ELSE 'primary' END " + "|| ':' || COALESCE((SELECT sync_state FROM pg_stat_replication " + "WHERE application_name='loom_standby'), 'absent')", + timeout=5, + cleanup=True, + ) + recovery, sync = value.split(":") + if recovery in {"primary", "standby"} and sync in { + "sync", + "async", + "potential", + "quorum", + "absent", + }: + row["pg_state"] = {"recovery": recovery == "standby", "standby_sync": sync} + except ( + OSError, + RuntimeError, + subprocess.SubprocessError, + ValueError, + KeyError, + ) as error: + row["diagnostic_error_type"] = type(error).__name__ + return {"phase": self.phase, "containers": observations} + def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: selected = ports(8) ch_port, native_port, pg_port, standby_port = selected[:4] @@ -349,8 +413,33 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "shared_buffers=128MB", ], ) - psql = ["docker", "exec", pg["id"], "psql", "-U", "loom", "-d", "loom", "-Atc"] - self.wait(lambda: self.command(*psql, "SELECT 1") == "1") + psql = [ + "docker", + "exec", + pg["id"], + "psql", + "-U", + "loom", + "-d", + "loom", + "-Atc", + ] + self.wait( + "primary_tcp", + lambda: self.command( + "docker", + "exec", + pg["id"], + "pg_isready", + "-U", + "loom", + "-d", + "loom", + "-h", + "127.0.0.1", + ), + ) + self.phase = "replication_configuration" self.command(*psql, "CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replicator'") self.command( "docker", @@ -405,6 +494,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ], ) self.wait( + "standby_sync", lambda: ( self.command( *psql, @@ -414,9 +504,10 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ), ) == "sync" - ) + ), ) self.wait( + "clickhouse_query", lambda: ( self.command( "docker", @@ -431,7 +522,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "SELECT 1", ) == "1" - ) + ), ) return ch, pg, standby, selected @@ -640,7 +731,14 @@ def dependency_receipt(root: Path, output: Path) -> dict: return {"path": str(output), "sha256": sha(output)} -def publish(folder: Path, destination: Path, error: BaseException | None) -> None: +def publish( + folder: Path, + destination: Path, + error: BaseException | None, + *, + phase: str, + diagnostic_error_type: str | None, +) -> None: destination.mkdir(mode=0o700) # Raw native logs/settings contain bearer and bootstrap credentials: never upload them. fixture = folder / "fixture" @@ -654,7 +752,7 @@ def publish(folder: Path, destination: Path, error: BaseException | None) -> Non path = fixture / name if path.is_file(): shutil.copyfile(path, destination / name) - for name in ("source-before.json", "source-after.json"): + for name in ("source-before.json", "source-after.json", "startup-diagnostics.json"): if (folder / name).is_file(): shutil.copyfile(folder / name, destination / name) intent_output = destination / "delivery-intent" @@ -672,6 +770,8 @@ def publish(folder: Path, destination: Path, error: BaseException | None) -> Non "accepted": error is None, "error_type": type(error).__name__ if error is not None else "", "failure": failure_site(error), + "phase": phase, + "diagnostic_error_type": diagnostic_error_type, "utc": datetime.datetime.now(datetime.UTC).isoformat(), }, ) @@ -768,9 +868,11 @@ def main() -> int: for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): signal.signal(signum, stores.interrupted) error = None + diagnostic_error_type = None cleanup_complete = False try: ch, primary, standby, selected = stores.start(args.clickhouse_image) + stores.phase = "dependency_metadata" resources(folder) dependencies = dependency_receipt(root, folder / "dependencies.json") from fixture import runtime @@ -815,10 +917,15 @@ def main() -> int: } write(folder / "spec.json", spec) require(not stores.cancelled, "Cancelled before native acceptance") + stores.phase = "native" require(runtime(spec) == 0, "Trigger acceptance did not pass") require(source(root) == before, "Acceptance source changed") except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught + try: + write(folder / "startup-diagnostics.json", stores.failure_diagnostics()) + except BaseException as diagnostic_error: # noqa: BLE001 — diagnostic failure cannot replace the selected first error. + diagnostic_error_type = type(diagnostic_error).__name__ finally: # Native ownership closes first; exact created containers are removed even on failure. for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): @@ -846,7 +953,13 @@ def main() -> int: except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught if error is None else error try: - publish(folder, Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", error) + publish( + folder, + Path(os.environ["RUNNER_TEMP"]) / "cell-trigger-artifacts", + error, + phase=stores.phase, + diagnostic_error_type=diagnostic_error_type, + ) except BaseException as caught: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. error = caught if error is None else error stores.terminal = cleanup_complete @@ -858,6 +971,8 @@ def main() -> int: "accepted": error is None, "error_type": type(error).__name__ if error is not None else "", "failure": failure_site(error), + "phase": stores.phase, + "diagnostic_error_type": diagnostic_error_type, } ) ) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 1d398f00da..3f1c78429a 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -11,6 +11,7 @@ import os import re import socket +import stat import struct import sys import uuid @@ -42,7 +43,22 @@ def protocol() -> dict: - return read(HERE / PROTOCOL, expected_sha256=PROTOCOL_SHA) + descriptor = os.open(HERE / PROTOCOL, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + need( + stat.S_ISREG(info.st_mode) + and info.st_uid == os.geteuid() + and info.st_mode & 0o022 == 0 + and info.st_size <= 65536, + "owned fixed protocol constant", + ) + raw = stream.read(65537) + need( + len(raw) <= 65536 and hashlib.sha256(raw).hexdigest() == PROTOCOL_SHA, + "fixed protocol bytes", + ) + return json_object(raw) def integer(value, low=0, high=2**32 - 1): diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index 38db700e6f..71dc49e22b 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -577,6 +577,7 @@ "namespace.mark_seen": ("dashboard",), "namespace.register": ( "sdk_method", + "sdk_example", "dashboard", ), "namespace.register_category": ( diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 3650641cd8..8cb90a343c 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -163,3 +163,166 @@ def check(version): for changed in ("3.13.0", "3.12.invalid", "3.12.999"): with pytest.raises(fixture.Refused, match="fixed isolated CPython"): check(changed) + + +@pytest.fixture +def protocol_reader(tmp_path, monkeypatch): + import sys + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + data = (Path(fixture.__file__).parent / fixture.PROTOCOL).read_bytes() + path = tmp_path / fixture.PROTOCOL + path.write_bytes(data) + path.chmod(0o644) + monkeypatch.setattr(capture, "HERE", tmp_path) + return capture, fixture, path, data + + +def test_committed_public_protocol_is_readable_without_private_receipt_mode(protocol_reader): + capture, _, _, _ = protocol_reader + assert capture.protocol()["schema"] == 1 + + +def test_public_protocol_keeps_hash_and_symlink_guards(protocol_reader): + capture, fixture, path, data = protocol_reader + path.write_bytes(data + b"\n") + with pytest.raises(fixture.Refused): + capture.protocol() + path.unlink() + target = path.with_name("target.json") + target.write_bytes(data) + target.chmod(0o644) + path.symlink_to(target) + with pytest.raises((OSError, fixture.Refused)): + capture.protocol() + + +def test_private_receipt_still_refuses_public_read_permissions(protocol_reader): + _, fixture, path, _ = protocol_reader + with pytest.raises(fixture.Refused, match="owned private stream"): + fixture.read(path) + + +def test_replication_waits_for_primary_tcp_after_temporary_socket_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingStores(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.tcp_attempts = 0 + self.tcp_ready = False + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "pg_isready" in argv: + assert argv[argv.index("-h") + 1] == "127.0.0.1" + self.tcp_attempts += 1 + if self.tcp_attempts == 1: + raise RuntimeError("simulated TCP startup refusal") + self.tcp_ready = True + return "127.0.0.1:5432 - accepting connections" + if "psql" in argv: + query = argv[-1] + if query == "SELECT 1": + return "1" # The temporary init server accepts Unix sockets only. + assert self.tcp_ready, "Replication configuration raced the temporary init server" + return "sync" if "SELECT sync_state" in query else "" + return "1" if "clickhouse-client" in argv else "network-id" + + stores = InitializingStores() + stores.start("test-image") + assert stores.tcp_attempts == 2 and stores.tcp_ready + + +def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): + import json + + class DiagnosticStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + assert cleanup and timeout == 5 + if argv[:2] == ("docker", "inspect"): + value = container() + value["Config"]["Env"] = ["TOKEN=private-value"] + value["State"] = { + "Status": "running", + "Running": True, + "ExitCode": 0, + "OOMKilled": False, + "Error": "private-log", + } + return json.dumps([value]) + return "primary:sync" + + stores = DiagnosticStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + stores.phase = "standby_sync" + stores.items = [{"role": "pg-primary", "name": "intent", "id": "own", "created": "birth"}] + result = stores.failure_diagnostics() + assert result == { + "phase": "standby_sync", + "containers": [ + { + "role": "pg-primary", + "available": True, + "status": "running", + "exit_code": 0, + "oom_killed": False, + "pg_state": {"recovery": False, "standby_sync": "sync"}, + } + ], + } + assert "private" not in json.dumps(result) + + +def test_diagnostic_failure_preserves_primary_result_and_runs_cleanup( + tmp_path, monkeypatch, capsys +): + import json + import sys + + from scripts.gates import cell_trigger_acceptance as creator + + cleaned = [] + + class FailingStores(Stores): + def start(self, image): + raise ValueError("private primary failure") + + def failure_diagnostics(self): + raise OSError("private diagnostic failure") + + def cleanup(self): + cleaned.append(True) + return [] + + root = Path(sys.executable).parent.parent.parent + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("GITHUB_WORKSPACE", str(root)) + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + for name in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "GITHUB_REPOSITORY"): + monkeypatch.setenv(name, "123") + monkeypatch.setattr(sys, "argv", ["creator", "--clickhouse-image", "test-image"]) + monkeypatch.setattr(creator, "Stores", FailingStores) + monkeypatch.setattr(creator, "source", lambda _: {"head": "a" * 40, "tree": "b" * 40}) + monkeypatch.setattr(creator, "source_hashes", lambda _: {}) + monkeypatch.setattr(creator, "resources", lambda _: None) + monkeypatch.setattr(creator.os, "umask", lambda _: 0) + monkeypatch.setattr(creator.signal, "signal", lambda *_: None) + assert creator.main() == 1 + result = json.loads(capsys.readouterr().out) + assert cleaned == [True] + assert result["error_type"] == "ValueError" and result["diagnostic_error_type"] == "OSError" + assert "private" not in json.dumps(result) + published = json.loads((tmp_path / "cell-trigger-artifacts/result.json").read_text()) + assert ( + published["error_type"] == "ValueError" and published["diagnostic_error_type"] == "OSError" + ) From 5762c94c60ec1b7d1d9811174e5b8bdb3a39fc01 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:11:09 +0000 Subject: [PATCH 124/181] Placement fence tests name the cell actor Keep queued-placement and backout assertions on the required directory.place call contract. Signed-off-by: zlareb1 --- tests/integration/test_cell_control_fence.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e14eab1220..14d469d743 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -468,7 +468,7 @@ async def wait_for_lock(key): "SELECT cell FROM loom_router.cells WHERE cell = 'first' FOR UPDATE" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) # The placement has its org lock and is queued at the cell row before claim. for _ in range(100): @@ -502,7 +502,7 @@ async def wait_for_lock(key): "SELECT pg_advisory_xact_lock($1, hashtext($2))", directory._PLACE_LOCK, "queued" ) placement = asyncio.create_task( - directory.place("queued", "first", app.placement_refusal) + directory.place("queued", "first", app.placement_refusal, actor="cell:first") ) await wait_for_lock(directory._PLACE_LOCK) await cell_control.backout(_owner(local_admin), call, original) @@ -661,7 +661,9 @@ async def call(path, body): ) # Actually attempt placement before the acknowledged close returns to cleanup. try: - await directory.place("between-close-and-cleanup", "first", app.placement_refusal) + await directory.place( + "between-close-and-cleanup", "first", app.placement_refusal, actor="cell:first" + ) except directory.PlacementRefused as exc: placement_error = str(exc) return answer From 79cd5467e4ebec8d3fd63791535a3a0c4ab1bb47 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:15:22 +0000 Subject: [PATCH 125/181] Trigger stores use the CI server configuration Mount the global custom-setting prefix under config.d and create the configured database, matching the existing ClickHouse CI helper. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 9 +++-- tests/unit/test_cell_trigger_ci_creator.py | 41 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 587c0b5c4f..43e9691ed3 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -348,11 +348,14 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: True, ) ) - for filename in ("log_queries_readonly.xml", "custom_settings_prefixes.xml"): + for filename, section in ( + ("log_queries_readonly.xml", "users.d"), + ("custom_settings_prefixes.xml", "config.d"), + ): mounts.append( ( self.root / "dev/clickhouse-ci" / filename, - "/etc/clickhouse-server/users.d/" + filename, + "/etc/clickhouse-server/" + section + "/" + filename, True, ) ) @@ -366,6 +369,8 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "-p", f"127.0.0.1:{native_port}:9000", "-e", + "CLICKHOUSE_DB=loom", + "-e", "CLICKHOUSE_USER=loom", "-e", "CLICKHOUSE_PASSWORD=loomdev", diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 8cb90a343c..a591c2bd95 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -243,6 +243,47 @@ def command(self, *argv, timeout=120, cleanup=False): assert stores.tcp_attempts == 2 and stores.tcp_ready +@pytest.mark.parametrize("setting", ["config_mount", "database"]) +def test_clickhouse_creation_uses_ci_server_configuration(tmp_path, monkeypatch, setting): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + + class CapturedCreation(Exception): + pass + + class CapturingStores(Stores): + def command(self, *argv, timeout=120, cleanup=False): + return "network-id" + + def create(self, role, args): + assert role == "clickhouse" + self.creation_args = args + raise CapturedCreation + + stores = CapturingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + with pytest.raises(CapturedCreation): + stores.start("test-image") + args = stores.creation_args + if setting == "config_mount": + mounts = [args[index + 1] for index, value in enumerate(args) if value == "-v"] + for filename, section in ( + ("custom_settings_prefixes.xml", "config.d"), + ("log_queries_readonly.xml", "users.d"), + ): + assert ( + str(tmp_path / "dev/clickhouse-ci" / filename) + + ":/etc/clickhouse-server/" + + section + + "/" + + filename + + ":ro" + ) in mounts + else: + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + assert "CLICKHOUSE_DB=loom" in environment + + def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): import json From c9758437b7c1415397fc3e404daf15965db80d79 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:24:02 +0000 Subject: [PATCH 126/181] Trigger ClickHouse runs as its data-directory owner Match the supported container UID and GID to the owned bind directory so startup keeps the nested dictionary mount read-only. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 6 ++++++ tests/unit/test_cell_trigger_ci_creator.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 43e9691ed3..52701f103b 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -330,6 +330,8 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: ch_data = self.folder / "clickhouse-data" ch_data.mkdir(mode=0o777) ch_data.chmod(0o777) + # Matching ownership avoids recursive chown through the read-only dictionary mount. + ch_owner = ch_data.stat() mounts = [ (ch_data, "/var/lib/clickhouse", False), (self.root / "deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3", True), @@ -371,6 +373,10 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: "-e", "CLICKHOUSE_DB=loom", "-e", + f"CLICKHOUSE_UID={ch_owner.st_uid}", + "-e", + f"CLICKHOUSE_GID={ch_owner.st_gid}", + "-e", "CLICKHOUSE_USER=loom", "-e", "CLICKHOUSE_PASSWORD=loomdev", diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index a591c2bd95..728132f87d 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -243,7 +243,7 @@ def command(self, *argv, timeout=120, cleanup=False): assert stores.tcp_attempts == 2 and stores.tcp_ready -@pytest.mark.parametrize("setting", ["config_mount", "database"]) +@pytest.mark.parametrize("setting", ["config_mount", "database", "data_owner"]) def test_clickhouse_creation_uses_ci_server_configuration(tmp_path, monkeypatch, setting): from scripts.gates import cell_trigger_acceptance as creator @@ -279,9 +279,18 @@ def create(self, role, args): + filename + ":ro" ) in mounts - else: + elif setting == "database": environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] assert "CLICKHOUSE_DB=loom" in environment + else: + environment = [args[index + 1] for index, value in enumerate(args) if value == "-e"] + owner = (tmp_path / "clickhouse-data").stat() + assert f"CLICKHOUSE_UID={owner.st_uid}" in environment + assert f"CLICKHOUSE_GID={owner.st_gid}" in environment + assert not any( + value.startswith(("CLICKHOUSE_RUN_AS_ROOT=", "CLICKHOUSE_DO_NOT_CHOWN=")) + for value in environment + ) def test_failure_diagnostics_expose_only_allowlisted_owned_state(tmp_path): From bc6bcd63cd92b9f8df551aa0d9febba867006c10 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:34:52 +0000 Subject: [PATCH 127/181] Trigger setup verifies the owned Keeper configuration Read the effective prefix from the exact verified container because system.server_settings omits this configuration key. Keep mismatched-prefix and changed-identity refusals. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 28 ++- tests/unit/test_cell_trigger_ci_creator.py | 168 ++++++++++++++++++ 2 files changed, 188 insertions(+), 8 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index a4c3661fb5..5f970cd051 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -909,14 +909,6 @@ async def stores(context: dict, role: str, output: Path, *, drop: bool = False) == [{"n": 0}], "new CH database", ) - prefix = await rows( - ch, "SELECT value FROM system.server_settings WHERE name='keeper_map_path_prefix'" - ) - need( - len(prefix) == 1 - and prefix[0]["value"].rstrip("/") == spec["ch"]["keeper_prefix"].rstrip("/"), - "actual Keeper prefix", - ) existing = await rows( ch, "SELECT name FROM system.zookeeper WHERE path={path:String}", @@ -1857,6 +1849,26 @@ def inspect_stores(self, *, cleanup: bool = False) -> None: == expected["boot_id"], "store process identity", ) + prefix = self.o.command( + "keeper-prefix", + [ + "docker", + "exec", + self.s["stores"]["clickhouse"]["container_id"], + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ], + self.env({}), + cleanup=cleanup, + ) + need( + prefix["log"].read_text().strip() == self.s["ch"]["keeper_prefix"], + "actual Keeper prefix", + ) def certificate(self) -> None: def openssl(*args): diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 728132f87d..b78cb245b8 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -376,3 +376,171 @@ def cleanup(self): assert ( published["error_type"] == "ValueError" and published["diagnostic_error_type"] == "OSError" ) + + +@pytest.fixture +def keeper_config_case(tmp_path, monkeypatch): + import json + import sys + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + monkeypatch.setitem(sys.modules, "fixture", fixture) + from scripts.gates.testsupport.cell_trigger import capture + + monkeypatch.setitem(sys.modules, "capture", capture) + from scripts.gates.testsupport.cell_trigger import native + + expected = { + "container_id": "owned-ch", + "created_at": "birth", + "host_pid": 123, + "memory_limit_bytes": 5 * fixture.GIB, + "server_limit_bytes": 9 * fixture.GIB // 2, + "shared_buffers_bytes": 128 * 1024**2, + "ports": [], + "disk_path": str(tmp_path), + "start_ticks": "start", + "uid": 1000, + "boot_id": "boot", + } + actual = { + "Id": "owned-ch", + "Created": "birth", + "State": {"Running": True, "Pid": 123}, + "HostConfig": {"Memory": 5 * fixture.GIB}, + "NetworkSettings": {"Ports": {}}, + "Mounts": [{"Source": str(tmp_path)}], + } + calls = [] + owner = SimpleNamespace(owned=SimpleNamespace(identity=lambda _: (1, "start"))) + owner.prefix = "/keeper_map_tables\n" + + def command(role, argv, env, *, seconds=120, cleanup=False): + calls.append((argv, cleanup)) + log = tmp_path / (role + ".log") + if argv == ["docker", "inspect", "owned-ch"]: + log.write_text(json.dumps([actual])) + else: + assert argv == [ + "docker", + "exec", + "owned-ch", + "clickhouse", + "extract-from-config", + "--config-file", + "/etc/clickhouse-server/config.xml", + "--key", + "keeper_map_path_prefix", + ] + log.write_text(owner.prefix) + return {"log": log} + + owner.command = command + original_stat, original_read = native.os.stat, Path.read_text + + def stat(path, *args, **kwargs): + if str(path) == "/proc/123": + return SimpleNamespace(st_uid=1000) + return original_stat(path, *args, **kwargs) + + def read_text(path, *args, **kwargs): + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(native.os, "stat", stat) + monkeypatch.setattr(Path, "read_text", read_text) + spec = { + "workdir": str(tmp_path), + "fixture_id": "fixture", + "stores": {"clickhouse": expected}, + "ch": {"keeper_prefix": "/keeper_map_tables"}, + "databases": {"ch_schema": "bench_schema"}, + "pg_standby_dsn": "fake", + } + instance = native.Fixture(spec, owner) + monkeypatch.setattr(instance, "env", lambda _: {}) + return native, instance, owner, actual, calls + + +@pytest.mark.asyncio +async def test_keeper_config_allows_creation_without_sql_server_setting( + keeper_config_case, monkeypatch +): + from contextlib import asynccontextmanager + + import asyncpg + + native, instance, _, _, calls = keeper_config_case + instance.inspect_stores() + commands = [] + queries = [] + + class Postgres: + async def fetchval(self, query, *args): + return { + "SELECT NOT pg_is_in_recovery()": True, + "SELECT pg_is_in_recovery()": True, + "SHOW server_version_num": "160000", + "SHOW max_prepared_transactions": "0", + "SHOW synchronous_standby_names": "FIRST 1 (loom_standby)", + "SELECT pg_size_bytes(current_setting('shared_buffers'))": 128 * 1024**2, + "SELECT EXISTS(SELECT FROM pg_stat_replication WHERE state='streaming' " + "AND sync_state='sync' AND flush_lsn IS NOT NULL)": True, + }[query] + + async def close(self, **kwargs): + assert kwargs["timeout"] > 0 + + class ClickHouse: + async def command(self, query): + commands.append(query) + + async def connect(*args, **kwargs): + return Postgres() + + @asynccontextmanager + async def connections(*args, **kwargs): + yield Postgres(), ClickHouse() + + async def rows(ch, query, parameters=None): + queries.append(query) + if "keeper_map_path_prefix" in query: + return [] # The real config key is absent from declared ServerSettings. + if "max_server_memory_usage" in query: + return [{"value": str(9 * native.GIB // 2)}] + if "count()" in query: + return [{"n": 0}] + if "toString(uuid)" in query: + return [{"id": "database-uuid"}] + assert "system.zookeeper" in query + return [] + + monkeypatch.setattr(asyncpg, "connect", connect) + monkeypatch.setattr(native, "connections", connections) + monkeypatch.setattr(native, "rows", rows) + instance.s["stores"]["pg_primary"] = {"shared_buffers_bytes": 128 * 1024**2} + instance.s["stores"]["pg_standby"] = {"shared_buffers_bytes": 128 * 1024**2} + await native.stores(instance.s, "ch_schema", instance.w / "created.json") + assert commands == ["CREATE DATABASE `bench_schema` ENGINE=Atomic"] + assert len(calls) == 2 and calls[-1][0][-1] == "keeper_map_path_prefix" + assert not any("keeper_map_path_prefix" in query for query in queries) + + +@pytest.mark.parametrize("cleanup", [False, True]) +def test_keeper_config_refuses_wrong_actual_prefix(keeper_config_case, cleanup): + native, instance, owner, _, calls = keeper_config_case + owner.prefix = "/different-prefix\n" + with pytest.raises(native.Refused, match="actual Keeper prefix"): + instance.inspect_stores(cleanup=cleanup) + assert calls[-1][1] is cleanup + + +def test_keeper_config_probe_follows_store_ownership_validation(keeper_config_case): + native, instance, _, actual, calls = keeper_config_case + actual["Created"] = "foreign-birth" + with pytest.raises(native.Refused, match="current owned container"): + instance.inspect_stores() + assert [argv for argv, _ in calls] == [["docker", "inspect", "owned-ch"]] From 30d57c3fcf0ce595959d09b91231e60c1f0786bf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 22:57:11 +0000 Subject: [PATCH 128/181] Worker crash checks report the unproved predicate Keep physical death, retirement and replacement checks unchanged while reporting bounded failure observations. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 47 +++++++++++++++------- tests/unit/test_docker_provider_smoke.py | 50 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 726582244e..bdf79baaf2 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -160,28 +160,36 @@ def _kill_worker(worker: dict, parent: dict) -> None: os.close(fd) -async def retired(client, captured: list[dict]) -> bool: +async def retired(client, captured: list[dict], *, observation: dict | None = None) -> bool: from server.insights.store.provider_participants import running_queries + def result(reason: str) -> bool: + if observation is not None: + observation["retirement"] = reason + return reason == "proved" + rows = await client.query_all( "SELECT toString(participant_id) AS participant_id, state, version, host_proof " "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [row["participant_id"] for row in captured]}, ) if len(rows) != len(captured): - return False + return result("row_count") expected = {row["participant_id"]: row for row in captured} for row in rows: old = expected[row["participant_id"]] - if ( - row["state"] != "retired" - or int(row["version"]) <= int(old["version"]) - or row["host_proof"] != old["host_proof"] - or not provider_process.has_certificate(row["host_proof"]) - or await running_queries(row["participant_id"], ch=client) != 0 - ): - return False - return True + if row["state"] != "retired": + return result("state") + if int(row["version"]) <= int(old["version"]): + return result("version") + if row["host_proof"] != old["host_proof"]: + return result("host_proof") + if not provider_process.has_certificate(row["host_proof"]): + return result("certificate") + running = await running_queries(row["participant_id"], ch=client) + if running != 0: + return result("queries_unknown" if running is None else "queries_running") + return result("proved") async def crash_worker(client) -> None: @@ -190,18 +198,27 @@ async def crash_worker(client) -> None: victim = before[0] _kill_worker(victim, parent) deadline = time.monotonic() + 240 + observation: dict = {"death": "not_checked", "retirement": "not_checked"} while time.monotonic() < deadline: assert _supervisor() == parent - if provider_process.proves_stopped(victim["host_proof"]) and await retired( - client, [victim] - ): + stopped = provider_process.proves_stopped(victim["host_proof"]) + observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} + if stopped and await retired(client, [victim], observation=observation): after = _workers(await _worker_rows(client), parent) old = {row["participant_id"] for row in before} new = {row["participant_id"] for row in after} + observation.update( + replacement_count=len(new), + survivors=len(old & new), + victim_absent=victim["participant_id"] not in new, + ) if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: return await asyncio.sleep(1) - raise AssertionError("Crashed worker retirement or supervised replacement was not proved") + raise AssertionError( + "Crashed worker retirement or supervised replacement was not proved: " + + json.dumps(observation, sort_keys=True) + ) async def seed() -> None: diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index b1a472c394..8b29bc9077 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -402,3 +402,53 @@ async def test_retirement_requires_exact_newer_certificate_and_known_zero_querie monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=count)) assert await data.retired(client, [rows[0]]) is (fault == "none") assert client.query_all.await_args.args[1] == {"ids": [rows[0]["participant_id"]]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault,reason", + [ + ("death_unknown", '"death": "unproved"'), + ("missing_rows", '"retirement": "row_count"'), + ("queries_unknown", '"retirement": "queries_unknown"'), + ("replacement_mismatch", '"victim_absent": false'), + ("none", None), + ], +) +async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, fault, reason): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from server.insights.store import provider_participants + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: fault != "death_unknown") + victim = rows[0] + replacement = {**victim, "participant_id": "replacement"} + after = rows if fault == "replacement_mismatch" else [rows[1], replacement] + monkeypatch.setattr(data, "_workers", lambda *_: after) + client = AsyncMock() + client.query_all.return_value = ( + [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}] + ) + monkeypatch.setattr( + provider_participants, + "running_queries", + AsyncMock(return_value=None if fault == "queries_unknown" else 0), + ) + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + if reason is None: + await data.crash_worker(client) + else: + with pytest.raises(AssertionError) as failure: + await data.crash_worker(client) + assert reason in str(failure.value) + assert victim["host_proof"] not in str(failure.value) + assert victim["participant_id"] not in str(failure.value) + assert signals == [victim] From f32c6f18f1dc716eb0d768d67c57d2ec043ca9db Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Tue, 17 Nov 2026 23:15:36 +0000 Subject: [PATCH 129/181] Attach fixtures isolate authority records and preserve failure causes Migrate a private ClickHouse database pair before attach tests so replacing authority rows cannot erase the integration server's deployment. Exercise the current remove command, roll back the expected SQL failure, and give the doctor a deterministic two-table inventory. Trigger verification names the first failed authority field without changing its comparisons. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 12 +-- tests/integration/test_cell_attach.py | 97 +++++++++++++++++-- .../integration/test_cell_org_triggers_pg.py | 3 +- tests/integration/test_cell_unset_gate_pg.py | 52 +++++++++- tests/unit/test_bench_cell_trigger_gate.py | 78 +++++++++++++++ tests/unit/test_integration_suite_targets.py | 72 ++++++++++++++ 6 files changed, 297 insertions(+), 17 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index e4a436b2ce..68512d0e9d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1316,14 +1316,14 @@ async def _verify( ) else: key = str(expected["key"]) + _require(head["memory_key"] == key, "authority API memory_key") + _require(json.loads(head["value"]) == _payload(key), "authority API value") + _require(head["version"] == expected["version"], "authority API version") _require( - head["memory_key"] == key - and json.loads(head["value"]) == _payload(key) - and head["version"] == expected["version"] - and head["content_sha256"] == expected["content_sha256"] - and head["status"] == "active", - "authority API payload/version", + head["content_sha256"] == expected["content_sha256"], + "authority API content_sha256", ) + _require(head["status"] == "active", "authority API status") _compare(head, projections[id], tuple(head), "full memory projection") key = str(head["memory_key"]) _require(key in documents, "search document identity") diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index a84474806e..15094d202a 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -15,11 +15,13 @@ import argparse import asyncio +import contextlib import hashlib import json import os +import sys import uuid -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterator from typing import Any import httpx @@ -29,12 +31,13 @@ from server.cli import cell_attach, cell_attach_pg, cell_prepare from server.cli.cell_ch import Refused from server.core import cell, cell_probe, cell_triggers, instances, pg -from server.core.ch import get_client +from server.core import ch as ch_core from server.core.schema import pg_migrate from server.router import app as router_app from server.router import directory from server.settings import settings from tests.integration._pg_database import create_database, drop_database +from tests.integration._resources import ch_client_from_env from tests.unit.cell_control_owner import owner as _owner OPERATOR = "lco_" + "2" * 128 @@ -71,8 +74,67 @@ async def _admin(sql: str, *args: Any) -> Any: await conn.close() +@pytest.fixture(scope="module") +def cell_clickhouse_databases() -> Iterator[tuple[str, str]]: + """Migrate one owned pair; no authority write can replace the suite server's row.""" + from server.core.schema import migrate + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + os.environ.get(name) or pytest.skip(f"{name} not set") + suffix = uuid.uuid4().hex + names = (f"loom_it_attach_{suffix}", f"loom_it_attacht_{suffix}") + created: list[str] = [] + admin = ch_client_from_env(database="") + error: BaseException | None = None + try: + for name in names: + admin.command(f"CREATE DATABASE {name}") + created.append(name) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(settings, "ch_database", names[0]) + patch.setattr(settings, "ch_telemetry_database", names[1]) + normal_client = migrate.client + bootstrap = normal_client() + patch.setattr( + migrate, "client", lambda **kwargs: normal_client(**kwargs) if kwargs else bootstrap + ) + try: + assert migrate.main([]) == 0 + finally: + if sys.exception() is None: + bootstrap.close() + else: + with contextlib.suppress(Exception): + bootstrap.close() + admin.close() + admin = None + yield names + except BaseException as caught: # noqa: BLE001 — exact owned cleanup retains the first failure. + error = caught + finally: + if admin is None: + try: + admin = ch_client_from_env(database="") + except BaseException as caught: # noqa: BLE001 — cleanup cannot replace the first failure. + error = error or caught + if admin is not None: + for name in reversed(created): + try: + admin.command(f"DROP DATABASE {name} SYNC") + except BaseException as caught: # noqa: BLE001 — attempt every exact owned cleanup. + error = error or caught + try: + admin.close() + except BaseException as caught: # noqa: BLE001 — close cannot replace the first failure. + error = error or caught + if error is not None: + raise error + + @pytest.fixture -async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: +async def cell_stores( + monkeypatch: pytest.MonkeyPatch, cell_clickhouse_databases: tuple[str, str] +) -> AsyncIterator[dict[str, Any]]: cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") @@ -92,6 +154,8 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str "router_operator_sha256": hashlib.sha256(OPERATOR.encode()).hexdigest(), "router_operator_token": OPERATOR, "ch_cluster": "", + "ch_database": cell_clickhouse_databases[0], + "ch_telemetry_database": cell_clickhouse_databases[1], "public_base_url": f"https://{HOST}", "oauth_issuer": "", }.items(): @@ -100,7 +164,8 @@ async def cell_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str monkeypatch.setattr(directory, "_pool", None) # A step's poll interval and the waits stay short. monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) - ch = get_client() + ch = ch_core.CHClient() + monkeypatch.setattr(ch_core, "_ch", ch) await ch.connect() ids: list[str] = [] real_live, real_recent = instances.read_live, instances.read_recent @@ -115,7 +180,7 @@ async def read_recent( rows = await real_recent(ch, window_s=window_s, since=since) return [r for r in rows if r["instance_id"] in ids] - # The integration server shares ClickHouse and reports its own process heartbeats. + # These assertions inspect only the heartbeat identities created by this fixture. monkeypatch.setattr(instances, "read_live", read_live) monkeypatch.setattr(instances, "read_recent", read_recent) try: @@ -763,6 +828,23 @@ async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( "INSERT INTO loom.it_large SELECT $1 || g FROM generate_series(1, 20000) g", tag + "l" ) await pg.execute("ANALYZE loom.it_small, loom.it_large") + # This ordering check owns two Postgres tables; shared ClickHouse logs keep growing. + monkeypatch.setattr( + cell_triggers, + "ORG_TABLES", + "SELECT relname FROM pg_class WHERE oid IN " + "('loom.it_small'::regclass, 'loom.it_large'::regclass)", + ) + + async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: + return [], [] + + monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) + clock = [0.0] + real_budget = cell_scan.Budget + monkeypatch.setattr( + cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + ) # The budget runs out on the largest table, as it does on a real cell when that table's # read is what is too slow: every smaller one was read first. real, read = cell_scan.pg_orgs, [] @@ -770,12 +852,13 @@ async def test_doctor_reads_smallest_first_and_leaves_the_largest_unchecked( async def pg_orgs(table: str, budget: Any) -> list[str]: read.append(table) if table == "loom.it_large": - await asyncio.sleep(budget.left() + 0.1) + clock[0] += budget.left() + 0.1 + budget.require(table) return await real(table, budget) monkeypatch.setattr(cell_scan, "pg_orgs", pg_orgs) lines = {n: (g, d) for n, g, d in await doctor_cell._stray_reads(stores["ch"], 3)} - assert read[-1] == "loom.it_large" and "loom.it_small" in read + assert read == ["loom.it_small", "loom.it_large"] assert tag + "small" in lines["cell_stray_orgs"][1] grade, detail = lines["cell_stray_orgs_unchecked"] assert grade == "WARN" and detail.endswith(": loom.it_large") diff --git a/tests/integration/test_cell_org_triggers_pg.py b/tests/integration/test_cell_org_triggers_pg.py index c219d05777..e8c797522e 100644 --- a/tests/integration/test_cell_org_triggers_pg.py +++ b/tests/integration/test_cell_org_triggers_pg.py @@ -672,7 +672,8 @@ async def test_a_dropped_guard_function_is_named_in_any_state_and_fails_an_enfor await conn.execute(f"DROP FUNCTION {cell_triggers.TENANCY_FUNCTION}()") if state == "enforced": with pytest.raises(asyncpg.exceptions.UndefinedFunctionError): - await cell_triggers.ensure(conn) + async with conn.transaction(): + await cell_triggers.ensure(conn) defects = {(d.kind, d.text) for d in await cell_triggers.missing(conn)} assert ("function", "loom.tenancy_refuse_unmarked() is absent") in defects if state == "enforced": # the failed boot created no guard diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index e94d5adb3f..37e80a7057 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -245,22 +245,68 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( ) -> None: """`loom cell remove`'s check of a first cell: one session from LOOM_PG_DSN, open and recording the row's deployment id, or it refuses.""" + import argparse + import asyncpg + import httpx from server.cli import cell as cell_cli + from server.cli.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) + monkeypatch.setattr(settings, "cell", "eu-1") + monkeypatch.setattr(settings, "document_s3_bucket", "") monkeypatch.setattr(pg, "_expected", None) + row = { + "cell": "eu-1", + "is_first": True, + "host": "cell.example.test", + "state": "attaching", + "placed_orgs": 0, + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + "probe_url": "", + "upstream_url": "", + "current_fp": None, + "next_fp": None, + "mutation_generation": "00000000-0000-0000-0000-000000000001", + } + calls = [] + + def router(request: httpx.Request) -> httpx.Response: + calls.append(request.method) + if request.method == "GET" or request.url.path.endswith("/claim"): + return httpx.Response(200, json=row) + assert request.method == "DELETE" + return httpx.Response(200, json={"cell": "eu-1", "removed": True}) + + args = argparse.Namespace(cell="eu-1") + url = httpx.URL("https://router.example.test") + transport = httpx.MockTransport(router) conn = await asyncpg.connect(dsn) try: await conn.execute( "INSERT INTO loom.authority (authority, epoch, deployment_id) " "VALUES ('postgres', 1, 'dep-1')" ) - assert await cell_cli._first_cell_refusal("dep-1") == "" - assert "records deployment id 'dep-1'" in await cell_cli._first_cell_refusal("dep-2") + assert await cell_cli._remove_locked(args, url, transport) == { + "cell": "eu-1", + "removed": True, + } + assert calls == ["GET", "POST", "DELETE"] + calls.clear() + row["deployment_id"] = "dep-2" + with pytest.raises(Refused, match="do not match the directory registration"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] + assert await conn.fetchval("SELECT deployment_id FROM loom.authority") == "dep-1" + calls.clear() + row["deployment_id"] = "dep-1" await _write_state(conn, {"tenancy"}) - assert "reads `partial`" in await cell_cli._first_cell_refusal("dep-1") + with pytest.raises(Refused, match="reads partial"): + await cell_cli._remove_locked(args, url, transport) + assert calls == ["GET"] finally: await conn.close() diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 7543067fd6..d7d59688e5 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -957,6 +957,84 @@ async def query(sql, params): ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["", "memory_key", "value", "version", "content_sha256", "status"] +) +async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fault): + import logging + + import server.embedding + from server.core.polled_copy import PolledCopy + from server.embedding.fake import FakeNGramEmbedder + from server.llm import customer + from server.memory.store import pg_memory + + monkeypatch.setattr( + customer, + "_switch", + PolledCopy("switch", AsyncMock(return_value=frozenset()), logging.getLogger(__name__)), + ) + monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) + block = await gate._build_block(args(), 1) + row, audit = block.rows[0], block.audits[0] + expected = gate.RequestRow( + str(row["memory_key"]), + str(row["memory_id"]), + row["version"], + row["content_sha256"], + 1.0, + True, + "derived_ready", + ) + head = dict(row) + if fault: + head[fault] = { + "memory_key": "changed-key", + "value": json.dumps("changed-value"), + "version": row["version"] + 1, + "content_sha256": "0" * 64, + "status": "conflicted", + }[fault] + stored = json.dumps(pg_memory.encode_row(head)) + conn = SimpleNamespace( + fetch=AsyncMock( + return_value=[ + { + "head": stored, + "revision": stored, + "audit": json.dumps(pg_memory.encode_audit(audit)), + "seq": 1, + "targets": 0, + "attempts": 0, + "last_error": "", + } + ] + ), + fetchrow=AsyncMock(return_value={"pending": 0, "audits": 0, "errors": 0}), + ) + document = await gate._expected_document(args(), row) + for field in ("embedding", "memory_key_embedding"): + document[field] = stored_bfloat16(document[field]) + + async def query(sql, params): + if "FROM loom.memory FINAL" in sql: + return [row] + if "FROM loom.memory_embeddings FINAL" in sql: + return [document] + if "FROM loom.audit_log" in sql: + return [audit] + raise AssertionError(sql) + + ch = SimpleNamespace(query_all=query) + if fault: + with pytest.raises(gate.Invalid) as raised: + await gate._verify(conn, ch, args(), [expected]) + assert str(raised.value) == "authority API " + fault + else: + await gate._verify(conn, ch, args(), [expected]) + + @pytest.mark.asyncio async def test_real_customer_policy_can_refuse_expected_vectors_without_bypass(monkeypatch): import logging diff --git a/tests/unit/test_integration_suite_targets.py b/tests/unit/test_integration_suite_targets.py index df7a98561a..1665edcd26 100644 --- a/tests/unit/test_integration_suite_targets.py +++ b/tests/unit/test_integration_suite_targets.py @@ -114,6 +114,78 @@ def test_the_gate_catches_each_implicit_target(tmp_path: Path) -> None: ], found +@pytest.mark.parametrize("fault", ["none", "second_create", "migration", "first_drop"]) +def test_cell_attach_owns_only_its_created_clickhouse_databases(monkeypatch, fault): + from server.core.schema import migrate + from server.settings import settings + from tests.integration import test_cell_attach as attach + + for name in ("LOOM_PG_DSN", "LOOM_ROUTER_PG_DSN", "LOOM_CH_HOST"): + monkeypatch.setenv(name, "owned-test-target") + initial = settings.ch_database, settings.ch_telemetry_database + created, dropped, connections, migrations = [], [], [], [] + + class Driver: + closed = False + + def command(self, sql): + assert not self.closed + action, _, name, *_ = sql.split() + if action == "CREATE": + if fault == "second_create" and created: + raise RuntimeError("second create refused") + created.append(name) + else: + assert action == "DROP" and name in created + dropped.append(name) + if fault == "first_drop" and len(dropped) == 1: + raise RuntimeError("first drop refused") + + def close(self): + self.closed = True + + def admin(**kwargs): + assert kwargs == {"database": ""} + driver = Driver() + connections.append(driver) + return driver + + bootstrap = Driver() + monkeypatch.setattr(attach, "ch_client_from_env", admin) + monkeypatch.setattr(migrate, "client", lambda **kwargs: bootstrap) + + def apply(args): + assert args == [] + assert migrate.client() is bootstrap + migrations.append((settings.ch_database, settings.ch_telemetry_database)) + assert migrations[-1] == tuple(created) + assert all(name.startswith("loom_it_attach") for name in created) + return 1 if fault == "migration" else 0 + + monkeypatch.setattr(migrate, "main", apply) + fixture = attach.cell_clickhouse_databases.__wrapped__() + if fault == "second_create": + with pytest.raises(RuntimeError, match="second create refused"): + next(fixture) + assert migrations == [] + elif fault == "migration": + with pytest.raises(AssertionError): + next(fixture) + else: + assert next(fixture) == tuple(created) + assert bootstrap.closed and all(driver.closed for driver in connections) + if fault == "first_drop": + with pytest.raises(RuntimeError, match="first drop refused"): + next(fixture) + else: + with pytest.raises(StopIteration): + next(fixture) + assert dropped == list(reversed(created)) + assert all(driver.closed for driver in connections) + assert (settings.ch_database, settings.ch_telemetry_database) == initial + assert len(migrations) == (0 if fault == "second_create" else 1) + + def test_the_gate_catches_a_clickhouse_cli_without_the_target_fixture(tmp_path: Path) -> None: probe = tmp_path / "test_probe.py" probe.write_text('CLI = ROOT / "dev" / "backup_loom.py"\n') From 7ad45ce88cf5af644750453ca138ebe67eff7dc8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 130/181] Attach acceptance verifies two genuine API replicas Join HTTP identities to live process and participant evidence, and finish owned startup and cleanup before store teardown. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + tests/integration/test_cell_attach.py | 87 +++- .../test_cell_attach_two_replicas.py | 492 ++++++++++++++++++ .../test_cell_attach_replica_ownership.py | 132 +++++ 4 files changed, 693 insertions(+), 19 deletions(-) create mode 100644 tests/integration/test_cell_attach_two_replicas.py create mode 100644 tests/unit/test_cell_attach_replica_ownership.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f3ed6e347..a96e01ebe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -910,6 +910,7 @@ jobs: tests/integration/test_cell_unset_gate_pg.py tests/integration/test_cell_prepare.py tests/integration/test_cell_attach.py + tests/integration/test_cell_attach_two_replicas.py tests/integration/test_cell_attach_cleanup.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 15094d202a..71a2b41b71 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -131,15 +131,30 @@ def cell_clickhouse_databases() -> Iterator[tuple[str, str]]: raise error +@pytest.fixture +def genuine_cell_processes() -> bool: + """Synthetic cases own their rows; the live-replica case uses the real readers.""" + return False + + @pytest.fixture async def cell_stores( - monkeypatch: pytest.MonkeyPatch, cell_clickhouse_databases: tuple[str, str] + monkeypatch: pytest.MonkeyPatch, + cell_clickhouse_databases: tuple[str, str], + genuine_cell_processes: bool, ) -> AsyncIterator[dict[str, Any]]: cell_dsn = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") router_dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") os.environ.get("LOOM_CH_HOST") or pytest.skip("LOOM_CH_HOST not set") cell_db, own_cell = await create_database(cell_dsn) - router_db, own_router = await create_database(router_dsn) + try: + router_db, own_router = await create_database(router_dsn) + except BaseException as first: # Release only the database already created by this fixture. + try: + await drop_database(cell_dsn, cell_db) + except BaseException as error: # noqa: BLE001 — cleanup retains the creation failure. + first.add_note(f"cell database cleanup also failed: {type(error).__name__}") + raise deployment = f"dep-{uuid.uuid4().hex[:8]}" tag = f"it-{uuid.uuid4().hex[:8]}-" for name, value in { @@ -160,13 +175,11 @@ async def cell_stores( "oauth_issuer": "", }.items(): monkeypatch.setattr(settings, name, value) - await pg.close_pool() monkeypatch.setattr(directory, "_pool", None) # A step's poll interval and the waits stay short. monkeypatch.setattr(cell_attach_pg, "POLL_S", 0.2) ch = ch_core.CHClient() monkeypatch.setattr(ch_core, "_ch", ch) - await ch.connect() ids: list[str] = [] real_live, real_recent = instances.read_live, instances.read_recent @@ -181,9 +194,12 @@ async def read_recent( return [r for r in rows if r["instance_id"] in ids] # These assertions inspect only the heartbeat identities created by this fixture. - monkeypatch.setattr(instances, "read_live", read_live) - monkeypatch.setattr(instances, "read_recent", read_recent) + if not genuine_cell_processes: + monkeypatch.setattr(instances, "read_live", read_live) + monkeypatch.setattr(instances, "read_recent", read_recent) try: + await pg.close_pool() + await ch.connect() await pg_migrate.migrate() await pg.execute( "INSERT INTO loom.authority (authority, epoch, deployment_id) " @@ -193,30 +209,63 @@ async def read_recent( await directory.migrate() await ch.insert( "loom.storage_authority", - [{"authority": "postgres", "epoch": 10**12, "deployment_id": deployment}], + [ + { + "authority": "postgres", + "epoch": 1 if genuine_cell_processes else 10**12, + "deployment_id": deployment, + } + ], ) cell.set_deployment_id(deployment) yield {"ch": ch, "ids": ids, "tag": tag, "deployment": deployment, "db": cell_db} finally: cell.set_deployment_id("") - await ch.execute( + await _close_cell_stores( + ch, deployment, ids, tag, (cell_dsn, cell_db), (router_dsn, router_db), sys.exception() + ) + + +async def _close_cell_stores( + ch: Any, + deployment: str, + ids: list[str], + tag: str, + cell_database: tuple[str, str], + router_database: tuple[str, str], + first: BaseException | None, +) -> None: + # Every operation owns an exact database, deployment, heartbeat id or org prefix. + cleanup = [ + ch.execute( "ALTER TABLE loom.storage_authority DELETE WHERE deployment_id = {d:String} " "SETTINGS mutations_sync = 2", {"d": deployment}, - ) - for table in (instances.TABLE, instances.CELL_TABLE): - await ch.execute( + ), + *( + ch.execute( f"ALTER TABLE {table} DELETE WHERE has({{ids:Array(String)}}, instance_id)", {"ids": ids}, ) - await ch.execute( - "DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag} - ) - await ch.disconnect() - await pg.close_pool() - await directory.close() - await drop_database(cell_dsn, cell_db) - await drop_database(router_dsn, router_db) + for table in (instances.TABLE, instances.CELL_TABLE) + ), + ch.execute("DELETE FROM loom.namespaces WHERE startsWith(org, {p:String})", {"p": tag}), + ch.disconnect(), + pg.close_pool(), + directory.close(), + drop_database(*cell_database), + drop_database(*router_database), + ] + for operation in cleanup: + try: + await operation + except BaseException as error: # noqa: BLE001 — attempt every owned cleanup; keep first failure. + if first is None: + first = error + else: + first.add_note(f"store cleanup also failed: {type(error).__name__}") + if first is not None: + raise first def _value(**keys: Any) -> str: diff --git a/tests/integration/test_cell_attach_two_replicas.py b/tests/integration/test_cell_attach_two_replicas.py new file mode 100644 index 0000000000..09a28fba88 --- /dev/null +++ b/tests/integration/test_cell_attach_two_replicas.py @@ -0,0 +1,492 @@ +"""Attach a first cell while two real API processes share its private stores. + +The router uses real HTTP upstreams and a test-owned, verified TLS listener. No +heartbeat row or health response is supplied by the test. Requires the same +synchronous Postgres and ClickHouse services as test_cell_attach. +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import ipaddress +import json +import os +import socket +import ssl +import subprocess +import sys +from collections.abc import AsyncIterator, Generator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from uuid import UUID + +import httpx +import pytest +import uvicorn +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + +from server.cli import cell_attach, cell_prepare +from server.core import cell_triggers, instances, pg, provider_process +from server.core.ch import query_tag +from server.insights.store import provider_participants +from server.router import app as router_app +from server.settings import settings +from tests.integration import test_cell_attach as attach_fixtures +from tests.integration._ports import free_port +from tests.integration._redact import redact +from tests.integration._resources import unique_org +from tests.integration._spawn import ROOT, loom_server +from tests.integration.test_cell_attach import ( + _args, + _mode, + _placed, + _row, +) + +_CREDENTIAL = "lcr_" + "3" * 128 +_WAIT_S = 45 + +cell_clickhouse_databases = attach_fixtures.cell_clickhouse_databases +cell_stores = attach_fixtures.cell_stores + + +@pytest.fixture +def genuine_cell_processes() -> bool: + return True + + +def _tls_files(root: Path) -> tuple[Path, Path, Path]: + """An ephemeral CA and IP-address leaf, trusted only by this test's clients.""" + now = datetime.now(UTC) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "attach test CA")]) + ca = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .sign(ca_key, hashes.SHA256()) + ) + key = ec.generate_private_key(ec.SECP256R1()) + leaf = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + ca_path, cert_path, key_path = (root / name for name in ("ca.pem", "server.pem", "key.pem")) + ca_path.write_bytes(ca.public_bytes(serialization.Encoding.PEM)) + cert_path.write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + key_path.chmod(0o600) + return ca_path, cert_path, key_path + + +class _RouterServer(uvicorn.Server): + @contextlib.contextmanager + def capture_signals(self) -> Generator[None]: + # pytest owns the main thread's signals. + yield + + +@contextlib.asynccontextmanager +async def _router(port: int, cert: Path, key: Path) -> AsyncIterator[None]: + router = router_app.Router() + server = _RouterServer( + uvicorn.Config( + router_app.build_app(router), + host="127.0.0.1", + port=port, + ssl_certfile=str(cert), + ssl_keyfile=str(key), + log_level="warning", + timeout_graceful_shutdown=5, + ) + ) + task = asyncio.create_task(server.serve(), name="attach-test-router") + try: + async with asyncio.timeout(10): + while not server.started: + if task.done(): + await task + raise AssertionError("router stopped before listening") + await asyncio.sleep(0.05) + yield + finally: + first = sys.exception() + server.should_exit = True + try: + await asyncio.wait_for(task, timeout=10) + except BaseException as error: # noqa: BLE001 — finish all owned cleanup; keep first failure. + first = first or error + finally: + # A timed-out uvicorn task must not leave its listener or requests running. + for listener in getattr(server, "servers", ()): + try: + listener.close() + await listener.wait_closed() + except BaseException as error: # noqa: BLE001 — attempt remaining owned cleanup. + first = first or error + requests = tuple(server.server_state.tasks) + for request in requests: + request.cancel() + for operation in ( + asyncio.wait_for(asyncio.gather(*requests, return_exceptions=True), timeout=5), + router.close(), + ): + try: + await operation + except BaseException as error: # noqa: BLE001 — keep first; finish router cleanup. + first = first or error + if first is not None: + raise first + + +@contextlib.asynccontextmanager +async def _api(name: str, env: dict[str, str], port: int) -> AsyncIterator[str]: + """Keep the router loop available and retain each worker until its context settles.""" + owned = loom_server(name, {**env, "LOOM_INSTANCE_LABEL": name}, port=port) + first: BaseException | None = None + + async def finish[T](task: asyncio.Task[T]) -> T: + nonlocal first + while not task.done(): + try: + await asyncio.wait({task}) + except asyncio.CancelledError as error: + if first is None: + first = error + return task.result() + + entry = asyncio.create_task(asyncio.to_thread(owned.__enter__)) + try: + yield await asyncio.shield(entry) + except BaseException as error: # noqa: BLE001 — cleanup completes before the first error escapes. + first = error + finally: + try: + await finish(entry) + except BaseException as error: # noqa: BLE001 — a failed entry already ran its own cleanup. + first = first or error + else: + try: + await finish( + asyncio.create_task(asyncio.to_thread(owned.__exit__, None, None, None)) + ) + except BaseException as error: # noqa: BLE001 — retain the body error after owned cleanup. + if first is None: + first = error + else: + first.add_note(f"{name} process cleanup also failed: {type(error).__name__}") + if first is not None: + raise first + + +def _configure(monkeypatch: pytest.MonkeyPatch, host: str, api: int, probe: int) -> dict[str, str]: + values = { + "cell": "first", + "storage_authority": "postgres", + "pg_durability": "synchronous", + "bind_host": "127.0.0.1", + "serves_traffic": True, + "pg_dsn": settings.pg_dsn, + "router_pg_dsn": settings.router_pg_dsn, + "ch_database": settings.ch_database, + "ch_telemetry_database": settings.ch_telemetry_database, + "ch_cluster": "", + "document_s3_bucket": "", + "public_base_url": f"https://{host}", + "oauth_issuer": "", + "cell_router_url": f"https://{host}", + "cell_router_token": _CREDENTIAL, + "cell_probe_port": probe, + "router_shared_host": host, + "router_upstream_url": f"http://127.0.0.1:{api}", + "router_upstream_probe_url": f"http://127.0.0.1:{probe}", + "instance_heartbeat_seconds": 1, + "tenancy_mode_cache_s": 1, + "router_directory_cache_s": 1, + "background_workers": "off", + "embedding_provider": "fake", + "embedding_model": "fake-ngram-1024", + "embedding_dim": 1024, + "llm_provider": "fake", + "llm_model": "fake-echo-1", + "require_real_providers": False, + "boot_migrate": False, + "auth_required": True, + "identity_provider": "gateway", + "self_signup": "off", + "mcp_auth_mode": "required", + } + for key, value in values.items(): + monkeypatch.setattr(settings, key, value) + env = { + f"LOOM_{key.upper()}": str(value).lower() if isinstance(value, bool) else str(value) + for key, value in values.items() + } + # API processes get the cell credential, never directory/operator authority. + env.update(LOOM_ROUTER_OPERATOR_TOKEN="", LOOM_ROUTER_OPERATOR_SHA256="", LOOM_ROUTER_PG_DSN="") + return env + + +async def _identities(client: httpx.AsyncClient, bases: list[str]) -> list[dict[str, Any]]: + identities = [] + for base in bases: + response = await client.get(f"{base}/v1/health") + assert response.status_code == 200, response.text + health = response.json() + host, pid = health["instance"].rsplit(":", 1) + assert host == socket.gethostname() + text = await asyncio.to_thread(Path(f"/proc/{int(pid)}/stat").read_text) + stat = text.rsplit(")", 1)[1].split() + assert int(stat[1]) == os.getpid(), "the API must be this test's child" + assert stat[0] not in {"Z", "X"} + identities.append( + { + "pid": int(pid), + "start": stat[19], + "participant": health["participant_id"], + "instance": health["deployment_config"]["instance_id"], + "label": health["instance_label"], + } + ) + assert len({i["pid"] for i in identities}) == 2 + assert len({i["participant"] for i in identities}) == 2 + assert len({i["instance"] for i in identities}) == 2 + assert {i["label"] for i in identities} == {"attach-two-a", "attach-two-b"} + return identities + + +async def _live(ch: Any, identities: list[dict[str, Any]], mode: str) -> None: + """Match unfiltered live and cell rows to the exact HTTP and kernel witnesses.""" + expected = {i["instance"] for i in identities} + async with asyncio.timeout(_WAIT_S): + while True: + rows = await instances.read_live(ch, all_deployments=True) + # KeeperMap PRIMARY KEY instance_id: one current row, no merge ordering. + cells = await ch.query_all(f"SELECT * FROM {instances.CELL_TABLE}") + by_id = {r["instance_id"]: r for r in cells} + if {r["instance_id"] for r in rows} == expected and all( + i["instance"] in by_id + and by_id[i["instance"]]["tenancy_mode"] == mode + and ( + mode != "enforced" + or next(r for r in rows if r["instance_id"] == i["instance"])["serving"] + ) + for i in identities + ): + break + await asyncio.sleep(0.2) + processes = await ch.query_all( + f"SELECT instance_id, pid FROM {instances.TABLE} " + "WHERE instance_id IN {ids:Array(String)}", + {"ids": sorted(expected)}, + ) + pids = {r["instance_id"]: int(r["pid"]) for r in processes} + boot = await asyncio.to_thread(Path("/proc/sys/kernel/random/boot_id").read_text) + participants = await provider_participants.participants(ch=ch) + by_participant = {str(r["participant_id"]): r for r in participants} + for identity in identities: + assert pids[identity["instance"]] == identity["pid"] + raw = by_id[identity["instance"]] + fingerprint = json.loads(raw["fingerprint"]) + assert raw["cell"] == "first" + assert fingerprint["participant_id"] == identity["participant"] + participant = by_participant[identity["participant"]] + assert participant["state"] == "running" + proof = participant["host_proof"] + assert provider_process.has_certificate(proof) + parts = proof.split("|") + assert parts[0] == socket.gethostname() + assert parts[1] == boot.strip() + assert parts[3] == str(os.stat("/proc/self/ns/pid").st_ino) + assert (parts[4], parts[5]) == (str(identity["pid"]), identity["start"]) + assert not provider_process.proves_stopped(proof) + if mode == "enforced": + assert fingerprint["router"] is True + assert fingerprint["marked"] is True + assert next(r for r in rows if r["instance_id"] == identity["instance"])["serving"] + identity["proof"] = proof + + +async def _retired(ch: Any, identities: list[dict[str, Any]]) -> None: + """A helper's TERM/KILL/reap alone does not establish guarded retirement.""" + rows = await ch.query_all( + f"SELECT participant_id, state, host_proof FROM {provider_participants.TABLE} FINAL " + "WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [UUID(i["participant"]) for i in identities]}, + ) + by_id = {str(row["participant_id"]): row for row in rows} + for identity in identities: + assert provider_process.proves_stopped(identity["proof"]) + row = by_id[identity["participant"]] + assert row["state"] == "retired", "stopping or killing is not retirement proof" + assert row["host_proof"] == identity["proof"] + assert provider_process.has_certificate(row["host_proof"]) + + +async def _mint(env: dict[str, str], org: str) -> str: + # A fresh native command reads this cell's keyring, not pytest's signing cache. + result = await asyncio.to_thread( + subprocess.run, + [ + sys.executable, + "-m", + "server.cli", + "mint-token", + "--subject", + "attach-two", + "--org", + org, + "--namespaces", + "n", + "--roles", + "writer", + "--ttl-days", + "1", + "--json", + ], + cwd=ROOT, + env={**os.environ, **env, "LOOM_CELL_ROUTER_TOKEN": ""}, + capture_output=True, + text=True, + timeout=_WAIT_S, + check=False, + ) + assert result.returncode == 0, redact(result.stderr) + token = json.loads(result.stdout)["token"] + assert isinstance(token, str) and token + return token + + +async def _requests(client: httpx.AsyncClient, bases: list[str], org: str, token: str) -> None: + headers = {"Authorization": f"Bearer {token}"} + key = {"org": org, "namespace": "n", "agent": "attach-test", "memory_key": "shared"} + for index, base in enumerate(bases): + unauthenticated = await client.post(f"{base}/v1/memory.get", json=key) + assert unauthenticated.status_code == 401, unauthenticated.text + value = {"writer": index} + written = await client.post( + f"{base}/v1/memory.set", + headers=headers, + json={**key, "memory_type": "semantic", "value": value}, + ) + assert written.status_code == 200, written.text + for reader in bases: + read = await client.post(f"{reader}/v1/memory.get", headers=headers, json=key) + assert read.status_code == 200, read.text + assert read.json()["value"] == value + + +@pytest.mark.asyncio +async def test_attach_with_two_live_api_replicas( + cell_stores: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + stores = cell_stores + api_a, api_b, probe, router_port = (free_port() for _ in range(4)) + host = f"127.0.0.1:{router_port}" + env = _configure(monkeypatch, host, api_a, probe) + ca, certificate, key = _tls_files(tmp_path) + env["SSL_CERT_FILE"] = str(ca) + trust = ssl.create_default_context(cafile=str(ca)) + router = httpx.URL(f"https://{host}") + org = unique_org("attach-two") + identities: list[dict[str, Any]] = [] + with query_tag("test.cell_attach_two_replicas"): + async with _router(router_port, certificate, key): + try: + async with ( + _api("attach-two-a", env, api_a) as a, + _api("attach-two-b", env, api_b) as b, + httpx.AsyncClient(verify=trust, timeout=30) as client, + ): + identities = await _identities(client, [a, b]) + stores["ids"].extend(i["instance"] for i in identities) + await _live(stores["ch"], identities, "open") + args = argparse.Namespace( + cell="first", + host=host, + credential_sha256=hashlib.sha256(_CREDENTIAL.encode()).hexdigest(), + probe_url="", + wait=20, + router=str(router), + ) + assert ( + await cell_prepare.prepare( + stores["ch"], args, router, httpx.AsyncHTTPTransport(verify=trust) + ) + == 0 + ) + await _live(stores["ch"], identities, "recording") + await pg.execute("INSERT INTO loom.tenancy (org) VALUES ($1)", org) + await stores["ch"].insert( + "loom.namespaces", [{"org": org, "namespace": "n", "owner_email": ""}] + ) + capsys.readouterr() + assert ( + await cell_attach.attach( + stores["ch"], + _args(router=str(router), wait=_WAIT_S), + router, + httpx.AsyncHTTPTransport(verify=trust), + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["state"] == "serving" + assert await _mode() == "enforced" + row = await _row() + assert (row.state, row.mode) == ("serving", "enforced") + assert org in await _placed() + async with pg.dedicated() as conn: + assert await cell_triggers.missing(conn) == [] + await _live(stores["ch"], identities, "enforced") + token = await _mint(env, org) + await _requests(client, [a, b, str(router)], org, token) + assert await _identities(client, [a, b]) == [ + {k: v for k, v in identity.items() if k != "proof"} + for identity in identities + ] + finally: + first = sys.exception() + if identities: + try: + await _retired(stores["ch"], identities) + except ( + BaseException + ) as error: # Retirement failure cannot hide the test failure. + if first is None: + raise + first.add_note( + f"retirement proof also failed: {type(error).__name__}: {error}" + ) diff --git a/tests/unit/test_cell_attach_replica_ownership.py b/tests/unit/test_cell_attach_replica_ownership.py new file mode 100644 index 0000000000..f2317d31a9 --- /dev/null +++ b/tests/unit/test_cell_attach_replica_ownership.py @@ -0,0 +1,132 @@ +"""The threaded API fixture owns entry and exit until cleanup has finished.""" + +import ast +import asyncio +import contextlib +import sys +import threading +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest + +SOURCE = Path(__file__).resolve().parents[1] / "integration/test_cell_attach_two_replicas.py" + + +def _fixture(context): + # Execute the actual helper alone: no server imports, services or subprocesses. + tree = ast.parse(SOURCE.read_text()) + helper = next(n for n in tree.body if isinstance(n, ast.AsyncFunctionDef) and n.name == "_api") + + def spawn(name, env, *, port): + assert (name, env, port) == ("owned", {"LOOM_INSTANCE_LABEL": "owned"}, 1) + return context + + namespace = { + "asyncio": asyncio, + "contextlib": contextlib, + "sys": sys, + "AsyncIterator": AsyncIterator, + "loom_server": spawn, + } + exec(compile(ast.Module(body=[helper], type_ignores=[]), str(SOURCE), "exec"), namespace) + return namespace["_api"] + + +@pytest.mark.parametrize("phase", ["entry", "exit", "body-error"]) +def test_cancellation_waits_for_owned_context_before_fixture_unwinds(phase): + async def scenario(): + observed = asyncio.Event() + entry_finished = threading.Event() + release = threading.Event() + loop = asyncio.get_running_loop() + unwound = [] + first = ValueError("body failed") if phase == "body-error" else None + + class Context: + alive = False + exits = 0 + + def __enter__(self): + self.alive = True + try: + if phase == "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release entry" + return "http://owned" + finally: + entry_finished.set() + + def __exit__(self, *_): + self.exits += 1 + if phase != "entry": + loop.call_soon_threadsafe(observed.set) + assert release.wait(5), "test did not release exit" + self.alive = False + + context = Context() + api = _fixture(context) + + async def owner(): + try: + async with api("owned", {}, 1) as base: + assert base == "http://owned" + if first is not None: + raise first + finally: + unwound.append(context.alive) + + task = asyncio.create_task(owner()) + try: + await asyncio.wait_for(observed.wait(), timeout=2) + for message in ("first cancel", "second cancel", "third cancel"): + task.cancel(message) + await asyncio.sleep(0) + assert not task.done(), "owner unwound while its context was still live" + assert unwound == [] and context.alive + release.set() + with pytest.raises( + ValueError if first is not None else asyncio.CancelledError + ) as caught: + await asyncio.wait_for(task, timeout=2) + if first is not None: + assert caught.value is first + else: + assert caught.value.args == ("first cancel",) + assert context.exits == 1 and not context.alive + assert unwound == [False] + finally: + release.set() + with contextlib.suppress(BaseException): + await task + assert await asyncio.to_thread(entry_finished.wait, 2) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("startup_fails", [False, True]) +def test_entry_outcome_selects_exactly_one_exit_or_its_existing_failure_cleanup(startup_fails): + first = ValueError("startup failed after its own cleanup") + exits = [] + + class Context: + def __enter__(self): + if startup_fails: + raise first + return "http://owned" + + def __exit__(self, *_): + exits.append(True) + + async def scenario(): + async with _fixture(Context())("owned", {}, 1) as base: + assert base == "http://owned" + + if startup_fails: + with pytest.raises(ValueError) as caught: + asyncio.run(scenario()) + assert caught.value is first + assert exits == [] + else: + asyncio.run(scenario()) + assert exits == [True] From 69b6efe7c03e8f00c75f73b0b756d443ca27d014 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 131/181] Worker crash checks read captured identities before signalling Refuse an unreadable worker before signalling and retain finite native-reader evidence when the original retirement proof fails. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 44 +++++++++++++-- tests/unit/test_docker_provider_smoke.py | 70 +++++++++++++++++++++++- 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index bdf79baaf2..c85f60c5bf 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -160,6 +160,30 @@ def _kill_worker(worker: dict, parent: dict) -> None: os.close(fd) +async def _retirement_rows(client, captured: list[dict]) -> list[dict]: + return await client.query_all( + "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + + +async def _native_retirement_observation(client, captured: list[dict]) -> dict: + try: + rows = await client.query_all( + "SELECT participant_id " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [row["participant_id"] for row in captured]}, + ) + return { + "reference_count": len(rows), + "reference_identity": {str(row["participant_id"]) for row in rows} + == {row["participant_id"] for row in captured}, + } + except Exception: # noqa: BLE001 — diagnostic failure cannot replace the unproved retirement + return {"reference_read": "query_error"} + + async def retired(client, captured: list[dict], *, observation: dict | None = None) -> bool: from server.insights.store.provider_participants import running_queries @@ -168,11 +192,7 @@ def result(reason: str) -> bool: observation["retirement"] = reason return reason == "proved" - rows = await client.query_all( - "SELECT toString(participant_id) AS participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", - {"ids": [row["participant_id"] for row in captured]}, - ) + rows = await _retirement_rows(client, captured) if len(rows) != len(captured): return result("row_count") expected = {row["participant_id"]: row for row in captured} @@ -196,6 +216,18 @@ async def crash_worker(client) -> None: parent = _supervisor() before = await workers(client) victim = before[0] + for worker in before: + rows = await _retirement_rows(client, [worker]) + identity_matches = len(rows) == 1 and rows[0]["participant_id"] == worker["participant_id"] + if not identity_matches: + readback = { + "reader_count": len(rows), + "reader_identity": identity_matches, + **await _native_retirement_observation(client, [worker]), + } + raise AssertionError( + "Captured worker readback failed: " + json.dumps(readback, sort_keys=True) + ) _kill_worker(victim, parent) deadline = time.monotonic() + 240 observation: dict = {"death": "not_checked", "retirement": "not_checked"} @@ -215,6 +247,8 @@ async def crash_worker(client) -> None: if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: return await asyncio.sleep(1) + if observation["retirement"] == "row_count": + observation.update(await _native_retirement_observation(client, [victim])) raise AssertionError( "Crashed worker retirement or supervised replacement was not proved: " + json.dumps(observation, sort_keys=True) diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 8b29bc9077..0122b28ed7 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -432,9 +432,12 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, after = rows if fault == "replacement_mismatch" else [rows[1], replacement] monkeypatch.setattr(data, "_workers", lambda *_: after) client = AsyncMock() - client.query_all.return_value = ( - [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}] - ) + client.query_all.side_effect = [ + [rows[0]], + [rows[1]], + [] if fault == "missing_rows" else [{**victim, "state": "retired", "version": 6}], + [victim], + ] monkeypatch.setattr( provider_participants, "running_queries", @@ -452,3 +455,64 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, assert victim["host_proof"] not in str(failure.value) assert victim["participant_id"] not in str(failure.value) assert signals == [victim] + reads = client.query_all.await_args_list + assert [call.args[1] for call in reads[:2]] == [ + {"ids": [row["participant_id"]]} for row in rows + ] + assert all( + "SELECT toString(participant_id) AS participant_id" in call.args[0] for call in reads[:2] + ) + if fault == "missing_rows": + assert '"reference_count": 1' in str(failure.value) + assert '"reference_identity": true' in str(failure.value) + assert "SELECT participant_id FROM" in reads[-1].args[0] + else: + assert all("SELECT participant_id FROM" not in call.args[0] for call in reads) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["missing-first", "missing-second", "wrong-identity", "reference-error"] +) +async def test_crash_refuses_unreadable_captured_worker_before_signalling(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, _processes, rows = _process_fixture(monkeypatch) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _: signals.append(victim)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: True) + client = AsyncMock() + + async def query(sql, parameters): + selected = next(row for row in rows if [row["participant_id"]] == parameters["ids"]) + if sql.startswith("SELECT participant_id FROM"): + if fault == "reference-error": + raise RuntimeError("private diagnostic failure") + return [selected] + if fault == "missing-second" and selected == rows[0]: + return [selected] + return [rows[1]] if fault == "wrong-identity" else [] + + client.query_all.side_effect = query + ticks = iter([0, 0, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", AsyncMock()) + with pytest.raises(AssertionError, match="Captured worker readback failed") as failure: + await data.crash_worker(client) + assert signals == [] + message = str(failure.value) + assert '"reader_identity": false' in message + assert f'"reader_count": {int(fault == "wrong-identity")}' in message + if fault == "reference-error": + assert '"reference_read": "query_error"' in message + assert "private diagnostic failure" not in message + else: + assert '"reference_count": 1' in message + assert '"reference_identity": true' in message + assert all(row["host_proof"] not in message for row in rows) + assert [call.args[1] for call in client.query_all.await_args_list] == ( + [{"ids": [rows[0]["participant_id"]]}] if fault == "missing-second" else [] + ) + [{"ids": [rows[int(fault == "missing-second")]["participant_id"]]}] * 2 From eadf2bf13038f7f8885cbb849fafa721c40f7f56 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 132/181] Standby drill checks ledger visibility after interrupted commit A commit waiting for its standby remains in the transaction snapshot. Keep the exact ledger assertion after cancellation releases the backend, and prove the index is ready beforehand. Signed-off-by: zlareb1 --- .../test_pg_online_index_durability.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py index f38453e747..5acb835065 100644 --- a/tests/integration/test_pg_online_index_durability.py +++ b/tests/integration/test_pg_online_index_durability.py @@ -131,14 +131,14 @@ async def pause_before_ledger(*args, **kwargs): first = asyncio.create_task(migrate()) tasks.append(first) pid = await _waiting(observer, first) - # The index is acknowledged already; its ledger COMMIT alone is unflushed. + # The index is acknowledged; the pending ledger transaction is not visible yet. async with connect() as conn: assert ( await conn.fetchval( - "SELECT count(*) FROM loom_router.pg_migrations " - "WHERE filename = '004_placements_by_cell_org.sql'" + "SELECT indisvalid AND indisready FROM pg_index " + "WHERE indexrelid = 'loom_router.placements_by_cell_org'::regclass" ) - == 1 + is True ) first.cancel() with pytest.raises(asyncio.CancelledError): @@ -152,6 +152,15 @@ async def pause_before_ledger(*args, **kwargs): await asyncio.sleep(0.01) else: pytest.fail("cancelled migration retained backend locks") + # Cancellation releases the commit wait before other sessions can see the ledger. + async with connect() as conn: + assert ( + await conn.fetchval( + "SELECT count(*) FROM loom_router.pg_migrations " + "WHERE filename = '004_placements_by_cell_org.sql'" + ) + == 1 + ) monkeypatch.setattr(pg_migrate, "_record_online", record) retry = asyncio.create_task(migrate()) tasks.append(retry) From bd11af3730fdc49d6f4e76eb3bd3d6b08eff720c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:04:01 +0000 Subject: [PATCH 133/181] Trigger acceptance keeps independent write heads stable Declare and verify the existing append-only policy in both arms while retaining exact row proofs and workload limits. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 7 ++ docs/benchmarks/results.md | 6 +- .../gates/testsupport/cell_trigger/native.py | 12 ++- tests/unit/test_bench_cell_trigger_gate.py | 12 +++ tests/unit/test_cell_trigger_ci_creator.py | 95 +++++++++++++++++++ 5 files changed, 128 insertions(+), 4 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 68512d0e9d..103eb8754d 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -3,6 +3,7 @@ Requires an independently owned, attached scratch fixture and its source/ownership manifest. The outer fixture owner stops its processes and drops its exact databases. This driver restores triggers but never deletes fixture rows or changes providers. +The namespace uses append_only so independent writes keep their exact heads. """ from __future__ import annotations @@ -634,6 +635,12 @@ async def _proof( "SELECT authority, epoch, deployment_id FROM loom.storage_authority FINAL" ) _require(len(records) == 1 and records[0] == dict(authority), "paired CH authority") + namespace = await ch.query_all( + "SELECT default_conflict_policy FROM loom.namespaces FINAL WHERE " + "org={org:String} AND namespace={ns:String} LIMIT 2", + {"org": args.org, "ns": args.namespace}, + ) + _require(namespace == [{"default_conflict_policy": "append_only"}], "independent write policy") for table in ("provider_configuration", "runtime_overrides"): rows = await ch.query_all(f"SELECT count() AS n FROM loom.{table}") _require(rows == [{"n": 0}], f"fresh {table}") diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index 00d4ca919b..fd940bf8ab 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -1244,7 +1244,11 @@ end-to-end gate is established by these measurements. `bench/cell_trigger_gate.py` measures the acceptance workload on an independently owned, attached scratch fixture with explicit fake providers, enforced mode and an already held -org. Each API run has 50 unmeasured warmups and 2,000 sequential measured requests with +org. Its namespace uses `append_only` for independent writes: sibling values do not +rewrite one another's heads. The fixture verifies the stored policy before every arm, +and the row proof requires each exact returned version and active status. This measures +trigger cost for independent writes, not semantic conflict resolution. Each API run has +50 unmeasured warmups and 2,000 sequential measured requests with 4,096-byte values, with five runs per arm in counterbalanced order. Each delivery block times five complete `pg_delivery.run_pass` calls at the unchanged configured batch size, with five blocks per arm. The latency gate is `median(p99_on) - median(p99_off) <= 10 ms`; diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 5f970cd051..c984661506 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1211,12 +1211,13 @@ async def proof( await rows( ch, ( - f"SELECT owner_email FROM `{db}`.namespaces FINAL WHERE " + "SELECT owner_email,default_conflict_policy " + f"FROM `{db}`.namespaces FINAL WHERE " f"org={{org:String}} AND namespace={{ns:String}}" ), params, ) - == [{"owner_email": "benchmark"}], + == [{"owner_email": "benchmark", "default_conflict_policy": "append_only"}], "native registered namespace", ) need( @@ -1731,7 +1732,12 @@ def credential(item, prefix): result = self.http( "/v1/namespace.register", token=self.token, - body={"org": self.org, "namespace": self.namespace, "owner_email": "benchmark"}, + body={ + "org": self.org, + "namespace": self.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + }, ) need( result.get("registered") is True and result.get("namespace") == self.namespace, diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index d7d59688e5..0cbd369201 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -627,6 +627,8 @@ async def query_all(self, sql, parameters=None): "epoch": 1, } ] + if "FROM loom.namespaces" in sql: + return [{"default_conflict_policy": "append_only"}] if "FROM loom.provider_configuration" in sql: return [{"n": int(self.fault == "saved_provider")}] if "FROM loom.runtime_overrides" in sql: @@ -1253,6 +1255,7 @@ async def send(self, request, **kwargs): "host", "instance", "uuid", + "namespace_policy", ], ) async def test_review_composed_proof_matches_actual_server_fingerprints_and_health( @@ -1276,6 +1279,15 @@ async def fetchrow(self, sql, *params): class CH(ProofCH): async def query_all(self, sql, params=None): + if "FROM loom.namespaces" in sql: + assert params == {"org": args().org, "ns": args().namespace} + return [ + { + "default_conflict_policy": "auto_reinforce" + if fault == "namespace_policy" + else "append_only" + } + ] if "FROM loom.instances" in sql: return [ { diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index b78cb245b8..0d412fb0d7 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -544,3 +544,98 @@ def test_keeper_config_probe_follows_store_ownership_validation(keeper_config_ca with pytest.raises(native.Refused, match="current owned container"): instance.inspect_stores() assert [argv for argv, _ in calls] == [["docker", "inspect", "owned-ch"]] + + +def test_native_setup_registers_independent_append_only_writes(keeper_config_case, monkeypatch): + native, instance, owner, _, _ = keeper_config_case + instance.s.update( + ports={"probe": 7790, "router": 7791, "tls": 7792, "api": 7793}, + pg_admin_dsn="postgresql://u:p@127.0.0.1:15432/postgres", + databases={ + "pg_cell": "bench_cell", + "pg_directory": "bench_directory", + "ch_schema": "bench_schema", + "ch_telemetry": "bench_telemetry", + }, + ch={"port": 18123, "username": "fixture", "password": "test-only"}, + head="a" * 40, + python="unused", + ) + log = instance.w / "credentials.log" + log.write_text("credential: lcr_" + "1" * 128 + "\ncredential: lco_" + "2" * 128 + "\n") + + def action(task, *, role="", **kwargs): + if task == "create": + native.write(instance.w / (role + "-created.json"), {"created": True}) + return {} + + class Registered(Exception): + pass + + def register(path, *, token, body): + assert path == "/v1/namespace.register" + assert token == "fixture-token" + assert body == { + "org": instance.org, + "namespace": instance.namespace, + "owner_email": "benchmark", + "default_conflict_policy": "append_only", + } + raise Registered + + monkeypatch.setattr(instance, "inspect_stores", lambda: None) + monkeypatch.setattr(instance, "action", action) + monkeypatch.setattr(instance, "cli", lambda *args, **kwargs: {"log": log}) + monkeypatch.setattr(instance, "command_receipt", lambda *args, **kwargs: None) + monkeypatch.setattr(instance, "certificate", lambda: None) + monkeypatch.setattr(instance, "start_api", lambda *args: None) + monkeypatch.setattr(instance, "process_receipt", lambda *args: {}) + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: {}, raising=False) + monkeypatch.setattr(native, "mint_metadata", lambda *args: ({}, "fixture-token")) + monkeypatch.setattr(instance, "http", register) + with pytest.raises(Registered): + instance.setup() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["auto_reinforce", "append_only"]) +async def test_native_bootstrap_checks_persisted_independent_write_policy( + keeper_config_case, monkeypatch, policy +): + from contextlib import asynccontextmanager + from types import SimpleNamespace + from unittest.mock import AsyncMock + + native, instance, _, _, _ = keeper_config_case + authority = {"authority": "postgres", "deployment_id": "deployment", "epoch": 1} + conn = SimpleNamespace( + fetchrow=AsyncMock(return_value=authority), fetchval=AsyncMock(return_value=0) + ) + + @asynccontextmanager + async def connected(*args, **kwargs): + yield conn, object() + + async def rows(ch, sql, params=None): + if ".storage_authority" in sql: + return [authority] + if ".namespaces " in sql: + assert params == {"org": instance.org, "ns": instance.namespace} + row = {"owner_email": "benchmark"} + if "default_conflict_policy" in sql: + row["default_conflict_policy"] = policy + return [row] + if ".namespace_grants " in sql: + return [{"subject": "benchmark", "role": "admin"}] + raise AssertionError(sql) + + monkeypatch.setattr(native, "connections", connected) + monkeypatch.setattr(native, "rows", rows) + output = instance.w / "bootstrap.json" + if policy != "append_only": + with pytest.raises(native.Refused, match="native registered namespace"): + await native.proof(instance.s, output, bootstrap=True) + assert not output.exists() + else: + await native.proof(instance.s, output, bootstrap=True) + assert native.read(output)["namespace_registered"] is True From 449769eb4ebe32f697d0d2beb36318f04a59d2df Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:19:34 +0000 Subject: [PATCH 134/181] Retirement readers filter the native participant UUID A same-name string projection masks the UUID filter in the native query. Read the native UUID and normalize its returned identity while retaining every retirement predicate. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 5 +++-- tests/unit/test_docker_provider_smoke.py | 26 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index c85f60c5bf..8c0104dd16 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -161,11 +161,12 @@ def _kill_worker(worker: dict, parent: dict) -> None: async def _retirement_rows(client, captured: list[dict]) -> list[dict]: - return await client.query_all( - "SELECT toString(participant_id) AS participant_id, state, version, host_proof " + rows = await client.query_all( + "SELECT participant_id, state, version, host_proof " "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [row["participant_id"] for row in captured]}, ) + return [{**row, "participant_id": str(row["participant_id"])} for row in rows] async def _native_retirement_observation(client, captured: list[dict]) -> dict: diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 0122b28ed7..53f02680b9 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -460,7 +460,7 @@ async def test_crash_observation_distinguishes_unproved_predicates(monkeypatch, {"ids": [row["participant_id"]]} for row in rows ] assert all( - "SELECT toString(participant_id) AS participant_id" in call.args[0] for call in reads[:2] + "SELECT participant_id, state, version, host_proof" in call.args[0] for call in reads[:2] ) if fault == "missing_rows": assert '"reference_count": 1' in str(failure.value) @@ -516,3 +516,27 @@ async def query(sql, parameters): assert [call.args[1] for call in client.query_all.await_args_list] == ( [{"ids": [rows[0]["participant_id"]]}] if fault == "missing-second" else [] ) + [{"ids": [rows[int(fault == "missing-second")]["participant_id"]]}] * 2 + + +@pytest.mark.asyncio +async def test_retirement_reader_preserves_string_identity_from_native_uuid(): + from unittest.mock import AsyncMock + from uuid import UUID + + participant_id = UUID("00112233-4455-6677-8899-aabbccddeeff") + row = { + "participant_id": participant_id, + "state": "retired", + "version": 6, + "host_proof": "proof", + } + client = AsyncMock() + client.query_all.return_value = [row] + captured = [{**row, "participant_id": str(participant_id)}] + assert await data._retirement_rows(client, captured) == captured + assert row["participant_id"] is participant_id + assert client.query_all.await_args.args == ( + "SELECT participant_id, state, version, host_proof " + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + {"ids": [str(participant_id)]}, + ) From 644f6531c9e742487caedae9b3ee1ff42357d90f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:39:25 +0000 Subject: [PATCH 135/181] The worker census skips exited supervisor children Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 12 ++++++- tests/unit/test_docker_provider_smoke.py | 40 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 8c0104dd16..bc476ee25b 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -87,7 +87,17 @@ def _kernel_workers() -> dict[int, dict]: children = Path("/proc/1/task/1/children").read_text().split() for value in children: pid = int(value) - process = _process(pid) + try: + process = _process(pid) + except FileNotFoundError: + # An exited child can remain in the supervisor's census until reaped. + try: + state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] + except FileNotFoundError: + continue + if state in {"Z", "X"}: + continue + raise command = process["command"] if not any( value.startswith("from multiprocessing.spawn import spawn_main;") for value in command diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 53f02680b9..1c7ffe3f18 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -540,3 +540,43 @@ async def test_retirement_reader_preserves_string_identity_from_native_uuid(): "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", {"ids": [str(participant_id)]}, ) + + +@pytest.mark.parametrize( + "fault", ["gone", "zombie", "dead", "live", "permission", "unreadable", "malformed", "one-live"] +) +def test_worker_census_omits_only_proved_exited_children(monkeypatch, fault): + from types import SimpleNamespace + + parent, _processes, rows = _process_fixture(monkeypatch) + process = data._process + + def observed_process(pid): + if pid == 7: + if fault == "permission": + raise PermissionError("unreadable child executable") + raise FileNotFoundError("disappeared child executable") + return process(pid) + + def read(path): + if path == "/proc/1/task/1/children": + return "7 8" if fault == "one-live" else "7 8 9" + assert path == "/proc/7/stat" + if fault == "gone": + raise FileNotFoundError("reaped child") + if fault == "unreadable": + raise PermissionError("unreadable child stat") + if fault == "malformed": + return "7 (incomplete child)" + state = "X" if fault == "dead" else "R" if fault == "live" else "Z" + return f"7 (exited child) {state} 1" + + monkeypatch.setattr(data, "_process", observed_process) + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + if fault in {"gone", "zombie", "dead"}: + assert data._workers(rows, parent) == [ + {key: row[key] for key in ("participant_id", "host_proof", "version")} for row in rows + ] + else: + with pytest.raises((OSError, IndexError, AssertionError)): + data._workers(rows, parent) From 88165daa343b43ad352fe4bbea0e5727aa855096 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:39:25 +0000 Subject: [PATCH 136/181] Trigger setup refusals retain bounded child diagnostics Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 100 ++++++++------- .../gates/testsupport/cell_trigger/native.py | 29 +++++ tests/unit/test_cell_trigger_ci_creator.py | 117 ++++++++++++++++++ 3 files changed, 203 insertions(+), 43 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 42526c9e43..b659c4b807 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -75,7 +75,8 @@ class Refused(RuntimeError): - pass + command: dict | None = None + native: dict | None = None def need(ok: object, field: str) -> None: @@ -568,6 +569,47 @@ def load_owned(path: Path): return OwnedChildren() +def error_metadata(error: BaseException) -> dict: + site = {"file": "unknown", "line": 0, "function": "unknown"} + frame = sys._getframe(1) + line = frame.f_lineno + trace = error.__traceback__ + for _ in range(64): + if trace is None: + break + candidate = trace.tb_frame + if Path(candidate.f_code.co_filename).name in FILES and candidate.f_code.co_name not in { + "need", + "closed", + "equal", + "integer", + "identifier", + "hex64", + }: + frame = candidate + line = trace.tb_lineno + trace = trace.tb_next + while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: + frame = frame.f_back + if frame is not None: + line = frame.f_lineno + if frame is not None and Path(frame.f_code.co_filename).name in FILES: + site = { + "file": Path(frame.f_code.co_filename).name, + "line": line, + "function": frame.f_code.co_name[:64], + } + return { + "error_type": type(error).__name__[:64], + "site": site, + "outcome": "refused" + if isinstance(error, Refused) + else "error" + if isinstance(error, Exception) + else "cancelled", + } + + class Owner: def __init__(self, spec: dict, owned, note) -> None: self.spec, self.owned, self.note = spec, owned, note @@ -601,47 +643,12 @@ def select(self, error: BaseException, *, phase: str | None = None) -> None: selected_phase = entry[1]["phase"] unknown = {"file": "unknown", "line": 0, "function": "unknown"} try: - site = unknown - frame = sys._getframe(1) - line = frame.f_lineno - trace = error.__traceback__ - for _ in range(64): - if trace is None: - break - candidate = trace.tb_frame - if Path( - candidate.f_code.co_filename - ).name in FILES and candidate.f_code.co_name not in { - "need", - "closed", - "equal", - "integer", - "identifier", - "hex64", - }: - frame = candidate - line = trace.tb_lineno - trace = trace.tb_next - while frame is not None and frame.f_code.co_name in {"select", "block_teardown"}: - frame = frame.f_back - if frame is not None: - line = frame.f_lineno - if frame is not None and Path(frame.f_code.co_filename).name in FILES: - site = { - "file": Path(frame.f_code.co_filename).name, - "line": line, - "function": frame.f_code.co_name[:64], - } - entry[1] = { - "phase": selected_phase, - "error_type": type(error).__name__[:64], - "site": site, - "outcome": "refused" - if isinstance(error, Refused) - else "error" - if isinstance(error, Exception) - else "cancelled", - } + entry[1] = {"phase": selected_phase, **error_metadata(error)} + if isinstance(error, Refused): + if error.command is not None: + entry[1]["command"] = error.command + if error.native is not None: + entry[1]["native"] = error.native except BaseException as secondary: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. entry[1] = { "phase": selected_phase, @@ -928,7 +935,14 @@ def command( self.monitor(cleanup=cleanup) need(time.monotonic() < end, "serial command deadline") time.sleep(0.05) - need(item["process"].returncode == 0, "native command refused") + if item["process"].returncode != 0: + error = Refused("native command refused") + error.command = { + "role": role if role in {"inspect", "keeper-prefix", "create"} else "other", + "serial": self.serial, + "returncode": item["process"].returncode, + } + raise error if self.first is not None and not cleanup: raise self.first return item diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index c984661506..e4fb7ed810 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -39,6 +39,7 @@ child_env, database, digest, + error_metadata, need, pg_dsn, read, @@ -1490,6 +1491,26 @@ def action( cleanup=cleanup, ) except BaseException as error: + if isinstance(error, Refused): + try: + failure = read(target.with_name(target.name + ".failure.json"), limit=4096) + closed(failure, "error_type site outcome") + site = failure["site"] + closed(site, "file line function") + need( + isinstance(failure["error_type"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", failure["error_type"]) + and failure["outcome"] in {"refused", "error", "cancelled"} + and site["file"] in (*FILES, "unknown") + and type(site["line"]) is int + and 0 <= site["line"] < 2**31 + and isinstance(site["function"], str) + and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", site["function"]), + "native failure metadata", + ) + error.native = failure + except (OSError, ValueError, Refused, KeyError, TypeError): + pass # Missing or invalid diagnostics cannot replace the command failure. # A terminal cleanup refusal is recorded, never retried as another mutation. if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: self.o.select(error) @@ -2400,6 +2421,14 @@ def main() -> int: ) ) except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. + try: + write( + args.output.with_name(args.output.name + ".failure.json"), + error_metadata(error), + limit=4096, + ) + except BaseException: # noqa: BLE001 — diagnostic output must not replace the native failure. + pass print(type(error).__name__, file=sys.stderr) return 1 return 0 diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 0d412fb0d7..1b6c0aa88f 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -639,3 +639,120 @@ async def rows(ch, sql, params=None): else: await native.proof(instance.s, output, bootstrap=True) assert native.read(output)["namespace_registered"] is True + + +@pytest.mark.parametrize("role", ["inspect", "keeper-prefix", "create", "private-role"]) +def test_nonzero_native_command_retains_only_structural_metadata(monkeypatch, role): + import json + from types import SimpleNamespace + + from scripts.gates.testsupport.cell_trigger import fixture + + owner = fixture.Owner({}, None, lambda *args, **kwargs: None) + owner.phase, owner.serial = "setup", 5 + item = {"process": SimpleNamespace(returncode=7)} + monkeypatch.setattr(owner, "start", lambda *args, **kwargs: item) + monkeypatch.setattr(owner, "poll", lambda _: 7) + with pytest.raises(fixture.Refused) as caught: + owner.command(role, ["private-argv"], {"TOKEN": "private-token"}) + owner.select(caught.value) + record = owner.failure_summary()["records"][0] + assert owner.first is caught.value + assert record["command"] == { + "role": role if role != "private-role" else "other", + "serial": 5, + "returncode": 7, + } + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "command" + assert "private-" not in json.dumps(record) + + +def test_native_failure_site_reaches_published_evidence_without_private_logs( + keeper_config_case, monkeypatch, tmp_path, capsys +): + import json + import sys + + from scripts.gates.cell_trigger_acceptance import publish + from scripts.gates.testsupport.cell_trigger import fixture + + native, instance, owner, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + instance.context = context + instance.s["databases"]["pg_cell"] = "bench_cell" + instance.s["python"] = "unused" + native.write(context, instance.s | {"root": str(tmp_path)}) + + async def failed(*args, **kwargs): + raise ValueError("private-native-secret") + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata["error_type"] == "ValueError" + assert metadata["site"]["file"] == "native.py" + assert metadata["site"]["function"] == "main" + assert metadata["site"]["line"] > 0 + assert "private-native-secret" not in json.dumps(metadata) + capsys.readouterr().err + original = fixture.Refused("private-command-secret") + original.command = {"role": "create", "serial": 5, "returncode": 1} + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(fixture.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + observer = fixture.Owner({}, None, lambda *args, **kwargs: None) + observer.select(caught.value, phase="setup") + evidence = observer.failure_summary() + assert evidence["records"][0]["native"] == metadata + assert evidence["records"][0]["command"] == original.command + folder = tmp_path / "publication" + (folder / "fixture").mkdir(parents=True) + native.write(folder / "fixture/outer-result.json", {"failure_evidence": evidence}) + (folder / "fixture/private.log").write_text("private-log-secret") + (folder / "fixture/private-settings.json").write_text('{"token":"private-settings-secret"}') + destination = tmp_path / "published" + publish(folder, destination, original, phase="native", diagnostic_error_type=None) + assert {path.name for path in destination.iterdir()} == {"outer-result.json", "result.json"} + published = json.loads((destination / "outer-result.json").read_text()) + assert published["failure_evidence"] == evidence + assert "private-" not in json.dumps(published) + + +@pytest.mark.parametrize("contents", ["not-json", '{"error_type":"private-extra"}', "x" * 4097]) +def test_invalid_native_failure_sidecar_preserves_command_refusal(keeper_config_case, contents): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + path = instance.w / "pg_cell-created.json.failure.json" + path.write_text(contents) + path.chmod(0o600) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None From 1a1cee2314c5ca206d54cb532e3bf0ba285099d4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:45:26 +0000 Subject: [PATCH 137/181] Diagnostic read failures preserve the original command refusal Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 4 +-- tests/unit/test_cell_trigger_ci_creator.py | 36 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index e4fb7ed810..9928315e56 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1509,8 +1509,8 @@ def action( "native failure metadata", ) error.native = failure - except (OSError, ValueError, Refused, KeyError, TypeError): - pass # Missing or invalid diagnostics cannot replace the command failure. + except BaseException: # noqa: BLE001 — diagnostics must not replace the original command failure. + pass # A terminal cleanup refusal is recorded, never retried as another mutation. if cleanup and self.o.leaders and self.o.leaders[-1]["terminal"] is not None: self.o.select(error) diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index 1b6c0aa88f..e1e77569c5 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -740,7 +740,15 @@ def refused(*args, **kwargs): assert "private-" not in json.dumps(published) -@pytest.mark.parametrize("contents", ["not-json", '{"error_type":"private-extra"}', "x" * 4097]) +@pytest.mark.parametrize( + "contents", + [ + "not-json", + '{"error_type":"private-extra"}', + "x" * 4097, + pytest.param("[" * 1500 + "0" + "]" * 1500, id="deep-nesting"), + ], +) def test_invalid_native_failure_sidecar_preserves_command_refusal(keeper_config_case, contents): native, instance, owner, _, _ = keeper_config_case instance.s["python"] = "unused" @@ -756,3 +764,29 @@ def refused(*args, **kwargs): with pytest.raises(native.Refused) as caught: instance.action("create", role="pg_cell") assert caught.value is original and original.native is None + + +@pytest.mark.parametrize("kind", [KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize("site", ["read", "closed"]) +def test_fatal_native_diagnostic_preserves_original_command_refusal( + keeper_config_case, monkeypatch, kind, site +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + def diagnostic(*args, **kwargs): + raise kind("private-diagnostic-secret") + + owner.command = refused + monkeypatch.setattr(native, "read", lambda *args, **kwargs: {}) + monkeypatch.setattr(native, site, diagnostic) + try: + instance.action("create", role="pg_cell") + except BaseException as error: # noqa: BLE001 — assert the actual retained failure, including fatal diagnostics. + assert error is original and original.native is None + else: + pytest.fail("command refusal was swallowed") From 1fcd50a4e08ef8814f5551d5ce0838c3a7e11521 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 00:58:16 +0000 Subject: [PATCH 138/181] The replica fixture matches its migrated vector width Signed-off-by: zlareb1 --- tests/integration/test_cell_attach_two_replicas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_cell_attach_two_replicas.py b/tests/integration/test_cell_attach_two_replicas.py index 09a28fba88..d9cb647db4 100644 --- a/tests/integration/test_cell_attach_two_replicas.py +++ b/tests/integration/test_cell_attach_two_replicas.py @@ -235,8 +235,8 @@ def _configure(monkeypatch: pytest.MonkeyPatch, host: str, api: int, probe: int) "router_directory_cache_s": 1, "background_workers": "off", "embedding_provider": "fake", - "embedding_model": "fake-ngram-1024", - "embedding_dim": 1024, + "embedding_model": "fake-ngram-3072", + "embedding_dim": 3072, "llm_provider": "fake", "llm_model": "fake-echo-1", "require_real_providers": False, From 6521d1b7a9dba8234495c7766b838d663b858100 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:16:46 +0000 Subject: [PATCH 139/181] Wait for supervised replacement enrollment after retirement Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 15 +++++- tests/unit/test_docker_provider_smoke.py | 64 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index bc476ee25b..9704e7c06f 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -30,6 +30,10 @@ ] +class _WorkersPending(AssertionError): + """A supervised replacement has not spawned or enrolled yet.""" + + def _process(pid: int) -> dict: root = Path(f"/proc/{pid}") stat = (root / "stat").read_text().rsplit(")", 1)[1].split() @@ -107,6 +111,8 @@ def _kernel_workers() -> dict[int, dict]: assert process["parent"] == 1, f"Kernel child PID {pid} parent changed during census" assert pid > 1 and pid not in found, "Invalid supervisor child PID census" found[pid] = process + if len(found) < 2: + raise _WorkersPending("Expected two live supervisor spawn children") assert len(found) == 2, f"Expected two live supervisor spawn children; found {len(found)}" return found @@ -127,6 +133,8 @@ def _workers(rows: list[dict], parent: dict) -> list[dict]: and proof[5] == process["start"] ): matches.append(row) + if not matches: + raise _WorkersPending(f"Worker PID {pid} exact registrations: 0; expected 1") assert len(matches) == 1, ( f"Worker PID {pid} exact registrations: {len(matches)}; expected 1" ) @@ -247,7 +255,12 @@ async def crash_worker(client) -> None: stopped = provider_process.proves_stopped(victim["host_proof"]) observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} if stopped and await retired(client, [victim], observation=observation): - after = _workers(await _worker_rows(client), parent) + try: + after = _workers(await _worker_rows(client), parent) + except _WorkersPending: + observation["replacement"] = "pending" + await asyncio.sleep(1) + continue old = {row["participant_id"] for row in before} new = {row["participant_id"] for row in after} observation.update( diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 1c7ffe3f18..64119f0817 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -580,3 +580,67 @@ def read(path): else: with pytest.raises((OSError, IndexError, AssertionError)): data._workers(rows, parent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", ["spawn", "registration", "timeout", "duplicate", "certificate", "namespace"] +) +async def test_crash_waits_only_for_incomplete_replacement_census(monkeypatch, fault): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + parent, processes, rows = _process_fixture(monkeypatch) + replacement = { + **rows[0], + "participant_id": "replacement", + "host_proof": "h|b|c|1|10|100|||cell-retirement-v1", + } + processes[10] = {**processes[8], "pid": 10, "start": "100"} + if fault == "certificate": + replacement["host_proof"] = "h|b|c|1|10|100|||unknown" + if fault == "namespace": + processes[10]["namespace"] = 2 + stage = 0 + + def children(_path): + pending = fault in {"spawn", "timeout"} and stage == 0 + return SimpleNamespace(read_text=lambda: "9" if pending else "9 10") + + async def registered(_client): + if fault == "registration" and stage == 0: + return [rows[1]] + if fault == "duplicate": + return [rows[1], replacement, {**replacement, "participant_id": "ambiguous"}] + return [rows[1], replacement] + + async def pause(_seconds): + nonlocal stage + stage += 1 + + monkeypatch.setattr(data, "Path", children) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + monkeypatch.setattr(data, "workers", AsyncMock(return_value=rows)) + monkeypatch.setattr(data, "_worker_rows", registered) + monkeypatch.setattr(data, "_retirement_rows", AsyncMock(side_effect=[[rows[0]], [rows[1]]])) + monkeypatch.setattr(data, "retired", AsyncMock(return_value=True)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _proof: True) + signals = [] + monkeypatch.setattr(data, "_kill_worker", lambda victim, _parent: signals.append(victim)) + ticks = iter([0, 0, 241] if fault == "timeout" else [0, 0, 1, 241]) + monkeypatch.setattr(data, "time", SimpleNamespace(monotonic=lambda: next(ticks))) + monkeypatch.setattr(data.asyncio, "sleep", pause) + if fault in {"spawn", "registration"}: + await data.crash_worker(object()) + assert stage == 1 + else: + pattern = { + "timeout": "supervised replacement was not proved", + "duplicate": "exact registrations: 2", + "certificate": "certificate invalid", + "namespace": "namespace mismatch", + }[fault] + with pytest.raises(AssertionError, match=pattern): + await data.crash_worker(object()) + assert stage == int(fault == "timeout") + assert signals == [rows[0]] From 3e8225721ed90cbd7d9d2b1f6136746329e2ce45 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:16:46 +0000 Subject: [PATCH 140/181] Attach deadline tests use an owned cleanup clock Signed-off-by: zlareb1 --- tests/unit/test_cell_attach.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 819f03d90f..ce341a619e 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -24,6 +24,7 @@ from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings from tests.unit.cell_control_owner import owner as _owner +from tests.unit.clock_fake import ManualClock FIELDS = {**cell.DEFAULTS, "deployment_id": "dep-1"} CURRENT = cell.as_value(FIELDS) @@ -531,6 +532,10 @@ def is_closed(self) -> bool: def _short_phases(monkeypatch: pytest.MonkeyPatch, bounds: list[float] | None = None) -> None: + # The work timer stays real; scheduling cannot consume the fake close budget. + phase_clock = ManualClock() + monkeypatch.setattr(cell_control, "time", phase_clock) + monkeypatch.setattr(cell_control.pg, "time", phase_clock) supervise = cell_control.Owner.call async def bounded(self: Any, work: Any, *, total_s: float) -> Any: From d5e14113b019b93e0ed7ab859a3bb4e7fd050bcb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:37:40 +0000 Subject: [PATCH 141/181] Resource samples require a complete process forest Restart an incomplete census only after a nonroot process is proven absent. Bound retries by the original freshness interval and retain identity, ownership and resource refusals. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 89 +++++-- .../unit/test_cell_trigger_resource_census.py | 242 ++++++++++++++++++ 2 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_cell_trigger_resource_census.py diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index b659c4b807..a279c450cb 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -745,24 +745,81 @@ def sample(self, *, admit: bool = False) -> None: and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], "resource store birth changed", ) - pending = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] - seen = set() - while pending: - pid = pending.pop() - if pid in seen: - continue - seen.add(pid) - before = self.owned.identity(pid) - need( - before is not None, "enumerated resource process disappeared; sample incomplete" - ) - need(self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], "process RSS") - pending.extend(self.owned.children(pid)) - after = self.owned.identity(pid) + roots = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] + births = {pid: self.owned.identity(pid) for pid in roots} + need(all(births.values()), "resource root disappeared") + observed = dict(births) + disappeared = set() + + def complete_forest() -> bool: + pending = list(roots) + seen = set() + while pending: + need(time.monotonic() - now < 3, "resource census stale") + pid = pending.pop() + if pid in seen: + continue + seen.add(pid) + before = self.owned.identity(pid) + if before is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need( + pid not in disappeared + and (pid not in observed or observed[pid][:2] == before[:2]), + "resource process birth changed", + ) + observed[pid] = before + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) + try: + children = self.owned.children(pid) + except (FileNotFoundError, ProcessLookupError): + need( + pid not in births and self.owned.identity(pid) is None, + "resource child census unknown", + ) + disappeared.add(pid) + return False + after = self.owned.identity(pid) + if after is None: + need(pid not in births, "resource root disappeared") + disappeared.add(pid) + return False + need(after[:2] == before[:2], "resource process birth changed") + pending.extend(children) + return True + + # Discard an incomplete traversal; only a fresh whole forest can renew the sample. + for _attempt in range(3): + need(time.monotonic() - now < 3, "resource census stale") + for pid, birth in births.items(): + current = self.owned.identity(pid) + need( + current is not None and current[:2] == birth[:2], + "resource root birth changed", + ) need( - after is not None and after[:2] == before[:2], - "resource process disappeared or changed; sample incomplete", + Path("/proc/sys/kernel/random/boot_id").read_text().strip() == boot + and os.stat(f"/proc/{self.owned.pid}").st_uid == os.geteuid(), + "resource owner birth changed", ) + for store in self.spec["stores"].values(): + need( + births[store["host_pid"]][1] == store["start_ticks"] + and Path("/proc/sys/kernel/random/boot_id").read_text().strip() + == store["boot_id"] + and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + "resource store birth changed", + ) + if complete_forest(): + need(time.monotonic() - now < 3, "resource census stale") + break + else: + raise Refused("resource process census remained incomplete") for item in tuple(self.leaders): need(item["log"].stat().st_size <= 16 * 1024 * 1024, "child log bound") driver_artifact = Path(self.spec["workdir"]) / "driver.jsonl" diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py new file mode 100644 index 0000000000..31a10c22cd --- /dev/null +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -0,0 +1,242 @@ +"""A vanished descendant requires a fresh complete resource census.""" + +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.gates.testsupport.cell_trigger import fixture + + +@pytest.fixture +def census(monkeypatch, tmp_path): + class Owned: + pid = 100 + + def __init__(self): + self.round = 0 + self.reads = Counter() + self.sampled_pids = [] + self.rows = {100: (1, "self", "S"), 200: (1, "store", "S")} + self.trees = [[300], [400]] + self.identify = lambda pid: self.rows.get(pid, (100, str(pid), "S")) + self.enumerate = lambda pid: ( + self.trees[min(self.round - 1, len(self.trees) - 1)] if pid == 100 else [] + ) + self.measure = lambda pid: 1 + + def identity(self, pid): + self.reads[pid] += 1 + return self.identify(pid) + + def children(self, pid): + if pid == self.pid: + self.round += 1 + return self.enumerate(pid) + + def rss(self, pid): + self.sampled_pids.append(pid) + return self.measure(pid) + + owned = Owned() + clock = SimpleNamespace(now=10.0) + original_read = Path.read_text + + def read(path, *args, **kwargs): + if str(path) == "/proc/meminfo": + return f"MemAvailable: {20 * 1024**2} kB\n" + if str(path) == "/proc/sys/kernel/random/boot_id": + return "boot" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(fixture, "time", SimpleNamespace(monotonic=lambda: clock.now)) + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr( + fixture.shutil, "disk_usage", lambda _: SimpleNamespace(free=32 * fixture.GIB) + ) + original_stat = fixture.os.stat + monkeypatch.setattr( + fixture.os, + "stat", + lambda path, *args, **kwargs: ( + SimpleNamespace(st_uid=fixture.os.geteuid()) + if str(path) in {"/proc/100", "/proc/200"} + else original_stat(path, *args, **kwargs) + ), + ) + spec = { + "workdir": str(tmp_path), + "stores": { + "pg": { + "host_pid": 200, + "start_ticks": "store", + "boot_id": "boot", + "uid": fixture.os.geteuid(), + "disk_path": str(tmp_path), + } + }, + } + owner = fixture.Owner(spec, owned, lambda *args, **kwargs: None) + owner.sampled = 7.0 + return owner, owned, clock + + +@pytest.mark.parametrize("stage", ["before", "after", "children"]) +def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300 and (stage != "after" or owned.reads[pid] > 1): + return None + return original_identity(pid) + + def children(pid): + if pid == 300 and stage == "children": + raise FileNotFoundError("gone") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if stage == "children": + owned.identify = lambda pid: ( + None if pid == 300 and owned.reads[pid] > 1 else original_identity(pid) + ) + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +def test_a_complete_first_forest_needs_no_retry(census): + owner, owned, _ = census + owner.sample() + assert owner.first is None and owner.sampled == 10.0 + assert owned.round == 1 and set(owned.sampled_pids) == {100, 200, 300} + + +@pytest.mark.parametrize( + "fault", + [ + "churn", + "root-gone", + "store-gone", + "store-birth", + "pid-reuse", + "identity-change", + "permission", + "malformed", + "rss", + "stale", + ], +) +def test_incomplete_or_unsafe_census_never_refreshes_the_sample(census, fault): + owner, owned, clock = census + original = owned.identify + + def identify(pid): + if fault == "root-gone" and pid == 100: + return None + if fault == "store-gone" and pid == 200: + return None + if fault == "store-birth" and pid == 200: + return (1, "changed", "S") + if pid >= 300: + if fault in {"churn", "stale"}: + if fault == "stale": + clock.now = 13.0 + return None + if fault == "pid-reuse" and owned.round == 1: + return None + if fault == "identity-change" and owned.reads[pid] > 1: + return (100, "new-birth", "S") + if fault == "permission": + raise PermissionError("unknown") + if fault == "malformed": + raise ValueError("unknown") + return original(pid) + + owned.identify = identify + if fault == "churn": + owned.trees = [[300], [400], [500], [600]] + if fault == "pid-reuse": + owned.trees = [[300], [300]] + if fault == "rss": + owned.measure = lambda pid: fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 3 + if fault not in {"churn", "pid-reuse", "stale"}: + assert owned.round <= 1 + + +@pytest.mark.parametrize( + "fault", ["root-birth", "store-birth", "new-child-rss", "live-child-census"] +) +def test_restarted_forest_retains_root_and_descendant_guards(census, fault): + owner, owned, _ = census + original_identity, original_children = owned.identify, owned.enumerate + + def identity(pid): + if pid == 300: + return original_identity(pid) if fault == "live-child-census" else None + if owned.round and ( + (pid == 100 and fault == "root-birth") or (pid == 200 and fault == "store-birth") + ): + return (1, "reused", "S") + return original_identity(pid) + + def children(pid): + if pid == 300 and fault == "live-child-census": + raise FileNotFoundError("unverified disappearance") + return original_children(pid) + + owned.identify, owned.enumerate = identity, children + if fault == "new-child-rss": + owned.measure = lambda pid: ( + fixture.POLICY["stop_process_rss_bytes"] // 1024 + 1 if pid == 400 else 1 + ) + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + if fault == "new-child-rss": + assert owned.round == 2 and 400 in owned.sampled_pids + else: + assert owned.round == 1 + + +def test_census_deadline_stops_before_the_next_descendant(census): + owner, owned, clock = census + original_children = owned.enumerate + owned.enumerate = lambda pid: [301] if pid == 300 else original_children(pid) + + def measure(pid): + if pid == 300: + clock.now = 13.0 + return 1 + + owned.measure = measure + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert 300 in owned.sampled_pids and 301 not in owned.sampled_pids + + +def test_restarted_forest_refuses_store_uid_drift(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + owned.identify = lambda pid: None if pid == 300 else original_identity(pid) + + def stat(path, *args, **kwargs): + if str(path) == "/proc/200" and owned.round: + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 From 3f69214963eb90835890e321d159809f3f25e415 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:46:13 +0000 Subject: [PATCH 142/181] Trigger protocol metadata describes its implemented runtime Keep every workload field and acceptance oracle unchanged while updating the fixed descriptor checksum. Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/fixture.py | 2 +- scripts/gates/testsupport/cell_trigger/protocol.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index a279c450cb..4862bf892e 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -39,7 +39,7 @@ HERE = Path(__file__).resolve().parent FILES = ("fixture.py", "native.py", "tls_proxy.py", "capture.py", "owned_children.py") PROTOCOL = "protocol.json" -PROTOCOL_SHA = "7223a2018c518cfc8a58977c84aa88dec5784aa741cb34245cb092abbf848812" +PROTOCOL_SHA = "690f1a06d0a523600a3ef4dcfa5cb58cc48052812ecad4e6e9e591c9e4582924" SPEC_FIELDS = { "schema", "fixture_id", diff --git a/scripts/gates/testsupport/cell_trigger/protocol.json b/scripts/gates/testsupport/cell_trigger/protocol.json index 3d3670f64d..850a6e3a27 100644 --- a/scripts/gates/testsupport/cell_trigger/protocol.json +++ b/scripts/gates/testsupport/cell_trigger/protocol.json @@ -1,6 +1,6 @@ { "schema": 1, - "status": "r5 design only; no runtime or entry implementation", + "status": "Runtime and entry implementation; native acceptance required", "memory_fields": [ "org", "namespace", From c9685ba94382169626258a852b7afdce1058c322 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:46:13 +0000 Subject: [PATCH 143/181] Phone citation geometry waits for the selected Ask option The option can be visible before its keyboard selection effect runs. Require that selection before the existing Enter action and retain every geometry assertion. Signed-off-by: zlareb1 --- frontend/e2e/assistant-launch.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/e2e/assistant-launch.spec.ts b/frontend/e2e/assistant-launch.spec.ts index f62187437d..ee9f2ad1df 100644 --- a/frontend/e2e/assistant-launch.spec.ts +++ b/frontend/e2e/assistant-launch.spec.ts @@ -369,7 +369,9 @@ test("on a phone the card's citation number keeps its glyph's width, and each so await page.getByRole("button", { name: "Search", exact: true }).tap(); const search = page.getByRole("combobox", { name: "Search Loom" }); await search.pressSequentially("how often are invoices sent?"); - await expect(page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' })).toBeVisible(); + const ask = page.getByRole("option", { name: 'Ask AI about "how often are invoices sent?"' }); + await expect(ask).toBeVisible(); + await expect(ask).toHaveAttribute("aria-selected", "true"); await page.keyboard.press("Enter"); const card = page.getByTestId("search-answer"); await expect(card.getByTestId("assistant-support")).toBeVisible(); From a400ba33a5e7d05bc000f7b55b6de5f228f4e9ea Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 01:52:12 +0000 Subject: [PATCH 144/181] Native trigger failures retain bounded driver diagnostics Keep numeric error codes and bounded cause classes without exception text or connection details. Validate the sidecar before attaching it to the original failure; execution and cleanup guards remain unchanged. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 63 ++++++++- ...est_cell_trigger_native_failure_details.py | 129 ++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_cell_trigger_native_failure_details.py diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 9928315e56..979d548483 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -94,6 +94,33 @@ ) +def native_failure_metadata(error: BaseException) -> dict: + metadata = error_metadata(error) + code = getattr(error, "code", None) + code = code if type(code) is int and 0 <= code < 2**31 else None + causes = [] + seen = {id(error)} + cause = error.__cause__ if error.__cause__ is not None else error.__context__ + for _ in range(4): + if cause is None or id(cause) in seen: + break + seen.add(id(cause)) + name = type(cause).__name__ + number = getattr(cause, "errno", None) + causes.append( + { + "error_type": name + if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) + else "unknown", + "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, + } + ) + cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ + if code is not None or causes: + metadata["driver"] = {"code": code, "causes": causes} + return metadata + + def pg_census_sql(): # One statement snapshot supplies all ten fixed counts and server observations. counts = [] @@ -1494,7 +1521,39 @@ def action( if isinstance(error, Refused): try: failure = read(target.with_name(target.name + ".failure.json"), limit=4096) - closed(failure, "error_type site outcome") + closed( + failure, + "error_type site outcome driver" + if "driver" in failure + else "error_type site outcome", + ) + if "driver" in failure: + driver = failure["driver"] + closed(driver, "code causes") + need( + ( + driver["code"] is None + or type(driver["code"]) is int + and 0 <= driver["code"] < 2**31 + ) + and type(driver["causes"]) is list + and len(driver["causes"]) <= 4, + "native driver failure metadata", + ) + for cause in driver["causes"]: + closed(cause, "error_type errno") + need( + isinstance(cause["error_type"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause["error_type"] + ) + and ( + cause["errno"] is None + or type(cause["errno"]) is int + and -(2**31) <= cause["errno"] < 2**31 + ), + "native driver failure cause", + ) site = failure["site"] closed(site, "file line function") need( @@ -2424,7 +2483,7 @@ def main() -> int: try: write( args.output.with_name(args.output.name + ".failure.json"), - error_metadata(error), + native_failure_metadata(error), limit=4096, ) except BaseException: # noqa: BLE001 — diagnostic output must not replace the native failure. diff --git a/tests/unit/test_cell_trigger_native_failure_details.py b/tests/unit/test_cell_trigger_native_failure_details.py new file mode 100644 index 0000000000..8f661ac541 --- /dev/null +++ b/tests/unit/test_cell_trigger_native_failure_details.py @@ -0,0 +1,129 @@ +"""Native failure details expose bounded numbers and classes without payloads.""" + +import json +import sys +from copy import deepcopy + +import pytest +from clickhouse_connect.driver.exceptions import OperationalError + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case + + +def detail(): + return { + "error_type": "OperationalError", + "site": {"file": "native.py", "line": 702, "function": "connections"}, + "outcome": "error", + "driver": { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + }, + } + + +@pytest.mark.parametrize("shape", ["transport", "bounded-cycle"]) +def test_native_sidecar_keeps_only_numeric_code_and_bounded_causes( + keeper_config_case, monkeypatch, tmp_path, shape +): + native, instance, _, _, _ = keeper_config_case + context, output = tmp_path / "failure-context.json", tmp_path / "pg_cell-created.json" + native.write( + context, + instance.s | {"root": str(tmp_path), "databases": {"pg_cell": "bench_cell"}}, + ) + monkeypatch.setattr(sys, "path", list(sys.path)) + primary = OperationalError("private-SQL-URL-settings-token", code=159) + cause = ConnectionRefusedError(111, "private-network-address") + primary.__cause__ = cause + expected = detail()["driver"] + if shape == "bounded-cycle": + previous = cause + for _ in range(6): + child = OSError(5, "private-payload") + previous.__cause__ = child + previous = child + previous.__cause__ = cause + expected["causes"] += [{"error_type": "OSError", "errno": 5}] * 3 + + async def failed(*args, **kwargs): + raise primary + + monkeypatch.setattr(native, "stores", failed) + monkeypatch.setattr( + sys, + "argv", + [ + "native.py", + "--task", + "create", + "--role", + "pg_cell", + "--context", + str(context), + "--output", + str(output), + ], + ) + assert native.main() == 1 + assert not output.exists() + raw = output.with_name(output.name + ".failure.json").read_bytes() + metadata = native.read(output.with_name(output.name + ".failure.json"), limit=4096) + assert metadata.get("driver") == expected + assert metadata["error_type"] == "OperationalError" + assert len(raw) <= 4096 and b"private-" not in raw + + +def test_valid_driver_detail_reaches_the_original_command_failure(keeper_config_case): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("private-command-failure") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native == metadata + assert "private-" not in json.dumps(original.native) + + +@pytest.mark.parametrize( + "field,value", + [ + ("code", True), + ("code", -1), + ("code", 2**31), + ("code", "private-code"), + ("causes", [{"error_type": "OSError", "errno": 5}] * 5), + ("class", "private-class-name"), + ("class", "A" * 65), + ("errno", True), + ("errno", 2**31), + ("extra", "private-message"), + ], +) +def test_unsafe_driver_detail_cannot_replace_the_command_failure(keeper_config_case, field, value): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = deepcopy(detail()) + driver = metadata["driver"] + if field in {"code", "causes"}: + driver[field] = value + elif field in {"class", "errno"}: + driver["causes"][0]["error_type" if field == "class" else "errno"] = value + else: + driver["message"] = value + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original") + + def refused(*args, **kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original and original.native is None From 24a159f743f89fcdfd36acc27b7bb5e42c6fea63 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 02:22:00 +0000 Subject: [PATCH 145/181] Provider smoke retains bounded reset failure evidence Collect read-only participant and configuration-lease counts before the failed fixture stack is removed. Keep the original activation failure and deadline when diagnostics fail or time out. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 69 +++++- scripts/gates/docker_provider_smoke.py | 23 +- .../test_docker_provider_reset_diagnostics.py | 220 ++++++++++++++++++ 3 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_docker_provider_reset_diagnostics.py diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 9704e7c06f..7f8611f850 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -9,6 +9,7 @@ import time from multiprocessing import spawn from pathlib import Path +from uuid import UUID from server.core import provider_process from server.core.ch import get_client @@ -340,11 +341,77 @@ async def verify(model: str, dimensions: int) -> None: print(f"PASS provider data: {model}/{dimensions}, memory + code metadata, no retained copies") +async def diagnostic(client, transition: str) -> None: + """Read capped aggregate evidence; never acknowledge or retire a participant.""" + from server.insights.store.provider_participants import PARTICIPANTS_SQL + + scope = provider_process.host_proof().split("|")[:4] + if len(scope) != 4 or not all(scope): + raise ValueError("Unavailable provider diagnostic scope") + sql = ( + "WITH retained AS (" + PARTICIPANTS_SQL + "), " + "lease AS (SELECT holder, renewed_at + toIntervalSecond(ttl_s) < now64(3) AS expired " + "FROM loom._migration_lock WHERE name = {name:String} LIMIT 2) " + "SELECT count() AS retained, " + "countIf(p.state = 'running') AS running, " + "countIf(p.state = 'drained' AND p.transition_id = {transition:UUID}) AS matching_drained, " + "countIf(p.state = 'drained' AND p.transition_id != {transition:UUID}) AS other_drained, " + "countIf(p.state = 'retired') AS retained_retired, " + "countIf(p.expired) AS expired, " + "countIf(arraySlice(splitByChar('|', p.host_proof), 1, 4) = {scope:Array(String)}) " + "AS same_scope, " + "(SELECT count() FROM lease) AS lease_rows, " + "(SELECT countIf(expired) FROM lease) AS expired_leases, " + "countIf(l.holder != '') AS retained_holders, " + "countIf(l.holder != '' AND p.state = 'drained' " + "AND p.transition_id = {transition:UUID}) AS matching_drained_holders " + "FROM retained AS p ANY LEFT JOIN lease AS l " + "ON toString(p.participant_id) = arrayElement(splitByChar('/', l.holder), 1) " + "SETTINGS max_execution_time = 3, max_rows_to_read = 10000, " + "max_bytes_to_read = 10485760, read_overflow_mode = 'throw', " + "max_result_rows = 1, max_result_bytes = 4096, result_overflow_mode = 'throw', " + "max_memory_usage = 134217728, max_threads = 1, select_sequential_consistency = 1" + ) + row = await asyncio.wait_for( + client.query_one( + sql, + { + "name": "provider_configuration", + "transition": str(UUID(transition)), + "scope": scope, + }, + ), + timeout=5, + ) + keys = { + "retained", + "running", + "matching_drained", + "other_drained", + "retained_retired", + "expired", + "same_scope", + "lease_rows", + "expired_leases", + "retained_holders", + "matching_drained_holders", + } + if ( + row is None + or row.keys() != keys + or any(type(value) is not int or not 0 <= value <= 10000 for value in row.values()) + ): + raise ValueError("Invalid provider diagnostic aggregate") + print("Provider reset diagnostic: " + json.dumps(row, sort_keys=True)) + + async def main() -> None: client = get_client() await client.connect() try: - if sys.argv[1] == "seed": + if sys.argv[1] == "diagnostic": + await diagnostic(client, sys.argv[2]) + elif sys.argv[1] == "seed": await seed() elif sys.argv[1] == "workers": print(json.dumps(await workers(client))) diff --git a/scripts/gates/docker_provider_smoke.py b/scripts/gates/docker_provider_smoke.py index b5445e145e..036da0dbc0 100644 --- a/scripts/gates/docker_provider_smoke.py +++ b/scripts/gates/docker_provider_smoke.py @@ -61,6 +61,16 @@ def control(compose: list[str], failed_model: str) -> None: def data_check(compose: list[str], *args: str) -> list[dict] | None: source = (ROOT / "scripts/gates/docker_provider_data.py").read_text() + if args[0] == "diagnostic": + subprocess.run( + [*compose, "exec", "-T", "loom", "python", "-", *args], + input=source, + text=True, + check=True, + timeout=10, + stderr=subprocess.DEVNULL, + ) + return None if args[0] == "workers": result = subprocess.run( [*compose, "exec", "-T", "loom", "python", "-", *args], @@ -203,9 +213,16 @@ def smoke_providers(base: str, compose: list[str]) -> None: data_check(compose, "docker-smoke-16", "16") reset = dashboard_action(base, "reset") assert reset["state"] == "queued", reset - status = wait_status( - base, lambda row: row["pending"] is None and row["source"] == "environment" - ) + try: + status = wait_status( + base, lambda row: row["pending"] is None and row["source"] == "environment" + ) + except BaseException: + try: + data_check(compose, "diagnostic", reset["config_id"]) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve activation failure + print("Provider reset diagnostic unavailable") + raise assert status["embedding_model"] == environment["embedding_model"], status assert status["llm_model"] == environment["llm_model"], status wait_admission(base) diff --git a/tests/unit/test_docker_provider_reset_diagnostics.py b/tests/unit/test_docker_provider_reset_diagnostics.py new file mode 100644 index 0000000000..36223ff9b7 --- /dev/null +++ b/tests/unit/test_docker_provider_reset_diagnostics.py @@ -0,0 +1,220 @@ +"""Failure evidence preserves the provider smoke's original reset refusal.""" + +import io +import json +import subprocess +import urllib.error +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +import scripts.gates.docker_provider_data as data +import scripts.gates.docker_provider_smoke as smoke + +TRANSITION = str(UUID(int=17)) + + +@pytest.mark.parametrize("diagnostic_failure", [None, RuntimeError, KeyboardInterrupt]) +def test_reset_failure_collects_diagnostic_and_preserves_first_error( + monkeypatch, tmp_path, diagnostic_failure +): + events, failure = _provider_flow(monkeypatch, tmp_path, diagnostic_failure=diagnostic_failure) + with pytest.raises(AssertionError) as caught: + smoke.smoke_providers("http://fixture", events["compose"]) + assert caught.value is failure + assert events["checks"][-1] == ("diagnostic", TRANSITION) + assert events["waits"] == 4 + assert events["resets"] == 1 + assert events["checks"].count(("crash-worker",)) == 1 + assert ( + "retired", + json.dumps([{"participant_id": "first"}, {"participant_id": "second"}]), + ) in events["checks"] + assert not any(args[0] == "environment" for args in events["checks"]) + + +def test_healthy_reset_does_not_collect_diagnostic(monkeypatch, tmp_path): + events, _ = _provider_flow(monkeypatch, tmp_path, healthy=True) + smoke.smoke_providers("http://fixture", events["compose"]) + assert not any(args[0] == "diagnostic" for args in events["checks"]) + assert events["checks"][-1] == ("environment", "8") + assert events["waits"] == 4 + assert events["resets"] == 1 + + +def _provider_flow(monkeypatch, tmp_path, *, diagnostic_failure=None, healthy=False): + override = tmp_path / "override.json" + override.write_text(json.dumps({"services": {"loom": {}}})) + events = { + "compose": ["docker", "compose", "-f", str(override)], + "checks": [], + "waits": 0, + "resets": 0, + } + failure = AssertionError("Timed out waiting for provider activation") + environment = { + "source": "environment", + "embedding_model": "environment", + "embedding_dimensions": 8, + "llm_model": "fake", + "pending": None, + } + active = { + "source": "dashboard", + "pending": None, + "embedding_model": "docker-smoke-16", + "embedding_dimensions": 16, + } + statuses = iter( + [ + { + **active, + "embedding_model": "docker-smoke-8", + "configured_keys": {"compatible": True}, + }, + { + "embedding_model": "docker-smoke-8", + "pending": {"state": "failed", "error_code": "rebuild_rebuild_failed"}, + }, + active, + environment, + ] + ) + model_reads = iter([environment, active]) + applications = iter(["queued", "active", "queued"]) + + def request(_base, path, _payload=None): + if path == "/v1/admin/models": + return next(model_reads) + raise urllib.error.HTTPError("http://fixture", 503, "closed", {}, None) + + def configure(*_args): + raise urllib.error.HTTPError( + "http://fixture", 422, "width", {}, io.BytesIO(b"unsupported_dimensions") + ) + + def dashboard(_base, action, *_args): + events["resets"] += action == "reset" + return { + "state": next(applications) if action == "apply" else "queued", + "config_id": TRANSITION, + } + + def check(_compose, *args): + events["checks"].append(args) + if args[0] == "diagnostic" and diagnostic_failure is not None: + raise diagnostic_failure("diagnostic secret must not replace the original") + return ( + [{"participant_id": "first"}, {"participant_id": "second"}] + if args[0] == "workers" + else None + ) + + def status(_base, predicate): + events["waits"] += 1 + value = next(statuses) + assert predicate(value) + if events["waits"] == 4 and not healthy: + raise failure + return value + + monkeypatch.setattr(smoke, "request", request) + monkeypatch.setattr(smoke, "configure", configure) + monkeypatch.setattr(smoke, "dashboard_action", dashboard) + monkeypatch.setattr(smoke, "data_check", check) + monkeypatch.setattr(smoke, "wait_status", status) + monkeypatch.setattr(smoke, "wait_admission", lambda *_: None) + monkeypatch.setattr(smoke, "control", lambda *_: None) + monkeypatch.setattr(smoke, "run", lambda *_: None) + return events, failure + + +def test_diagnostic_exec_is_bounded_and_discards_stderr(monkeypatch): + calls = [] + monkeypatch.setattr( + smoke.subprocess, "run", lambda *args, **kwargs: calls.append((args, kwargs)) + ) + smoke.data_check(["docker", "compose", "-f", "owned.json"], "diagnostic", TRANSITION) + args, kwargs = calls[0] + assert args[0][-2:] == ["diagnostic", TRANSITION] + assert kwargs["timeout"] == 10 + assert kwargs["check"] is True + assert kwargs["stderr"] == subprocess.DEVNULL + assert "capture_output" not in kwargs + + +@pytest.mark.asyncio +async def test_diagnostic_emits_only_closed_bounded_counts(monkeypatch, capsys): + from types import SimpleNamespace + + row = _counts() + query = AsyncMock(return_value=row) + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + await data.diagnostic(SimpleNamespace(query_one=query), TRANSITION) + sql, params = query.await_args.args + assert sql.lstrip().startswith("WITH retained AS (SELECT") + assert "ANY LEFT JOIN" in sql and "LIMIT 2" in sql + for guard in ( + "max_execution_time = 3", + "max_rows_to_read = 10000", + "max_bytes_to_read = 10485760", + "max_result_rows = 1", + "max_result_bytes = 4096", + "max_memory_usage = 134217728", + ): + assert guard in sql + assert params == { + "name": "provider_configuration", + "transition": TRANSITION, + "scope": ["host", "boot", "container", "namespace"], + } + output = capsys.readouterr().out + assert json.loads(output.split(": ", 1)[1]) == row + assert len(output) < 1024 + assert "host|boot" not in output and TRANSITION not in output + assert query.await_count == 1 + + +@pytest.mark.parametrize("fault", ["extra", "bool", "large", "negative", "missing", "none"]) +@pytest.mark.asyncio +async def test_diagnostic_refuses_invalid_aggregate_without_output(monkeypatch, capsys, fault): + from types import SimpleNamespace + + row = _counts() + if fault == "extra": + row["payload"] = "secret" + elif fault == "bool": + row["retained"] = True + elif fault == "large": + row["retained"] = 10001 + elif fault == "negative": + row["retained"] = -1 + elif fault == "missing": + del row["retained"] + else: + row = None + monkeypatch.setattr( + data.provider_process, "host_proof", lambda: "host|boot|container|namespace|55|90" + ) + with pytest.raises(ValueError, match="Invalid provider diagnostic aggregate"): + await data.diagnostic(SimpleNamespace(query_one=AsyncMock(return_value=row)), TRANSITION) + assert capsys.readouterr().out == "" + + +def _counts(): + return { + "retained": 4, + "running": 1, + "matching_drained": 3, + "other_drained": 0, + "retained_retired": 0, + "expired": 1, + "same_scope": 3, + "lease_rows": 1, + "expired_leases": 0, + "retained_holders": 1, + "matching_drained_holders": 1, + } From 2460fa2ff8298e957d7a5a0f852d385e811077e8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:04:54 +0000 Subject: [PATCH 146/181] Trigger acceptance retains bounded Postgres failure evidence Preserve the wrapped cause, error identifiers and commit uncertainty when staging refuses, without changing workload, retry or acceptance rules. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 15 +- .../gates/testsupport/cell_trigger/fixture.py | 54 +++- .../gates/testsupport/cell_trigger/native.py | 74 +++--- ...est_cell_trigger_driver_failure_details.py | 242 ++++++++++++++++++ 4 files changed, 349 insertions(+), 36 deletions(-) create mode 100644 tests/unit/test_cell_trigger_driver_failure_details.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 103eb8754d..18237c300c 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -34,6 +34,7 @@ import clickhouse_connect import httpx +from scripts.gates.testsupport.cell_trigger.fixture import FILES, native_failure_metadata from server.core import cell_triggers from server.core.hashing import hash_text, sha256_value @@ -1603,16 +1604,28 @@ async def run(ch: Any) -> dict[str, Any]: except BaseException as error: # noqa: BLE001 — every failed run must emit INVALID # Never serialize exception text: clients may include credentials or values. reason = str(error) if isinstance(error, Invalid) else "operation refused" + failure = None + try: + failure = native_failure_metadata( + error, files=(*FILES, "cell_trigger_gate.py", "pg.py", "pg_memory.py") + ) + except BaseException: # noqa: BLE001 — diagnostic failure must preserve INVALID + pass verdict = { "verdict": "INVALID", "phase": phase, "reason": reason, "error_type": type(error).__name__, + **({"failure": failure} if failure is not None else {}), } if artifact is not None: try: artifact.emit( - "invalid", phase=phase, reason=reason, error_type=type(error).__name__ + "invalid", + phase=phase, + reason=reason, + error_type=type(error).__name__, + **({"failure": failure} if failure is not None else {}), ) except BaseException: # noqa: BLE001 — secondary output cannot replace INVALID. pass diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 4862bf892e..27c280092c 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -569,16 +569,18 @@ def load_owned(path: Path): return OwnedChildren() -def error_metadata(error: BaseException) -> dict: +def error_metadata( + error: BaseException, *, files: tuple[str, ...] = FILES, fallback: bool = True +) -> dict: site = {"file": "unknown", "line": 0, "function": "unknown"} - frame = sys._getframe(1) - line = frame.f_lineno + frame = sys._getframe(1) if fallback else None + line = frame.f_lineno if frame is not None else 0 trace = error.__traceback__ for _ in range(64): if trace is None: break candidate = trace.tb_frame - if Path(candidate.f_code.co_filename).name in FILES and candidate.f_code.co_name not in { + if Path(candidate.f_code.co_filename).name in files and candidate.f_code.co_name not in { "need", "closed", "equal", @@ -593,7 +595,7 @@ def error_metadata(error: BaseException) -> dict: frame = frame.f_back if frame is not None: line = frame.f_lineno - if frame is not None and Path(frame.f_code.co_filename).name in FILES: + if frame is not None and Path(frame.f_code.co_filename).name in files: site = { "file": Path(frame.f_code.co_filename).name, "line": line, @@ -610,6 +612,48 @@ def error_metadata(error: BaseException) -> dict: } +def native_failure_metadata(error: BaseException, *, files: tuple[str, ...] = FILES) -> dict: + metadata = error_metadata(error, files=files) + if not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", metadata["error_type"]): + metadata["error_type"] = "unknown" + code = getattr(error, "code", None) + code = code if type(code) is int and 0 <= code < 2**31 else None + causes = [] + seen = {id(error)} + cause = error.__cause__ if error.__cause__ is not None else error.__context__ + for _ in range(4): + if cause is None or id(cause) in seen: + break + seen.add(id(cause)) + name = type(cause).__name__ + number = getattr(cause, "errno", None) + causes.append( + { + "error_type": name + if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) + else "unknown", + "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, + } + ) + cause_site = error_metadata(cause, files=files, fallback=False)["site"] + if cause_site["file"] != "unknown": + causes[-1]["site"] = cause_site + sqlstate = getattr(cause, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + causes[-1]["sqlstate"] = sqlstate + cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ + driver = {"code": code, "causes": causes} + sqlstate = getattr(error, "sqlstate", None) + if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): + driver["sqlstate"] = sqlstate + extra = getattr(error, "extra", None) + if type(extra) is dict and type(extra.get("outcome_unknown")) is bool: + driver["outcome_unknown"] = extra["outcome_unknown"] + if code is not None or causes or len(driver) > 2: + metadata["driver"] = driver + return metadata + + class Owner: def __init__(self, spec: dict, owned, note) -> None: self.spec, self.owned, self.note = spec, owned, note diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 979d548483..8d68c386db 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -39,7 +39,7 @@ child_env, database, digest, - error_metadata, + native_failure_metadata, need, pg_dsn, read, @@ -94,33 +94,6 @@ ) -def native_failure_metadata(error: BaseException) -> dict: - metadata = error_metadata(error) - code = getattr(error, "code", None) - code = code if type(code) is int and 0 <= code < 2**31 else None - causes = [] - seen = {id(error)} - cause = error.__cause__ if error.__cause__ is not None else error.__context__ - for _ in range(4): - if cause is None or id(cause) in seen: - break - seen.add(id(cause)) - name = type(cause).__name__ - number = getattr(cause, "errno", None) - causes.append( - { - "error_type": name - if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]{0,63}", name) - else "unknown", - "errno": number if type(number) is int and -(2**31) <= number < 2**31 else None, - } - ) - cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ - if code is not None or causes: - metadata["driver"] = {"code": code, "causes": causes} - return metadata - - def pg_census_sql(): # One statement snapshot supplies all ten fixed counts and server observations. counts = [] @@ -1529,7 +1502,24 @@ def action( ) if "driver" in failure: driver = failure["driver"] - closed(driver, "code causes") + closed( + driver, + "code causes" + + (" sqlstate" if "sqlstate" in driver else "") + + (" outcome_unknown" if "outcome_unknown" in driver else ""), + ) + need( + ( + "sqlstate" not in driver + or type(driver["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", driver["sqlstate"]) + ) + and ( + "outcome_unknown" not in driver + or type(driver["outcome_unknown"]) is bool + ), + "native Postgres failure metadata", + ) need( ( driver["code"] is None @@ -1541,7 +1531,31 @@ def action( "native driver failure metadata", ) for cause in driver["causes"]: - closed(cause, "error_type errno") + closed( + cause, + "error_type errno" + + (" sqlstate" if "sqlstate" in cause else "") + + (" site" if "site" in cause else ""), + ) + need( + "sqlstate" not in cause + or type(cause["sqlstate"]) is str + and re.fullmatch(r"[0-9A-Z]{5}", cause["sqlstate"]), + "native Postgres failure cause", + ) + if "site" in cause: + cause_site = cause["site"] + closed(cause_site, "file line function") + need( + cause_site["file"] in (*FILES, "unknown") + and type(cause_site["line"]) is int + and 0 <= cause_site["line"] < 2**31 + and isinstance(cause_site["function"], str) + and re.fullmatch( + r"[A-Za-z_][A-Za-z_0-9]{0,63}", cause_site["function"] + ), + "native failure cause site", + ) need( isinstance(cause["error_type"], str) and re.fullmatch( diff --git a/tests/unit/test_cell_trigger_driver_failure_details.py b/tests/unit/test_cell_trigger_driver_failure_details.py new file mode 100644 index 0000000000..ff41e5e20e --- /dev/null +++ b/tests/unit/test_cell_trigger_driver_failure_details.py @@ -0,0 +1,242 @@ +"""The native driver retains finite Postgres failure evidence without payloads.""" + +import asyncio +import dataclasses +import json +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +import bench.cell_trigger_gate as gate +from tests.unit.test_bench_cell_trigger_gate import args, fixture_config +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_native_failure_details import detail + + +@pytest.mark.parametrize("fault", ["query_cancel", "connection", "unknown_commit"]) +def test_actual_stage_failure_retains_pg_cause_in_invalid_result( + monkeypatch, tmp_path, capsys, fault +): + output, calls = _main_case(monkeypatch, tmp_path, fault) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" + assert verdict["phase"] == "native_driver" + assert verdict["reason"] == "operation refused" + assert verdict["error_type"] == "PostgresUnavailableError" + assert calls == ["wire", "stage", "native_returned"] + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + assert "failure" in verdict + metadata = verdict["failure"] + assert metadata["error_type"] == "PostgresUnavailableError" + assert metadata["site"]["file"] == "pg.py" + assert metadata["site"]["function"] == "_raise_if_unavailable" + assert metadata["site"]["line"] > 0 + assert metadata["driver"]["outcome_unknown"] is (fault == "unknown_commit") + expected = ( + {"error_type": "ConnectionResetError", "errno": 104} + if fault == "connection" + else {"error_type": "QueryCanceledError", "errno": None, "sqlstate": "57014"} + ) + assert metadata["driver"]["causes"] == [expected] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and invalid["failure"] == metadata + + +def test_healthy_native_driver_keeps_pass_without_failure_metadata(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, None) + assert gate.main() == 0 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "PASS" and "failure" not in verdict + assert calls == ["wire", "native_returned"] + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + + +def _main_case(monkeypatch, tmp_path, fault): + import server.settings + import server.wiring + from server.cli import _ch + from server.core import pg + from server.memory.store import pg_memory + + fixture, config = fixture_config() + manifest = tmp_path / "manifest" + manifest.write_text(json.dumps(dataclasses.asdict(fixture))) + output = tmp_path / "driver.jsonl" + argv = ["driver"] + for name, value in { + **vars(args()), + "fixture_manifest": str(manifest), + "output": str(output), + }.items(): + argv += ["--" + name.replace("_", "-"), str(value)] + calls = [] + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.setattr(server.settings, "settings", config) + monkeypatch.setattr(server.wiring, "wire", lambda: calls.append("wire")) + monkeypatch.setattr(gate, "_preflight", AsyncMock()) + monkeypatch.setattr(gate, "_source", lambda _: {"clean": True}) + monkeypatch.setattr(gate, "_launch", lambda *_: {}) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + + async def commit(*_args, **_kwargs): + calls.append("stage") + cause = ( + ConnectionResetError(104, "secret-trigger-diagnostic-payload") + if fault == "connection" + else asyncpg.QueryCanceledError("secret-trigger-diagnostic-payload") + ) + pg._raise_if_unavailable(cause, outcome_unknown=fault == "unknown_commit") + raise AssertionError("the actual PG wrapper must refuse") + + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + def native(*_args, **_kwargs): + try: + if fault: + block = gate.Block( + rows=[ + { + "org": "owned", + "namespace": "owned", + "memory_id": "00000000-0000-0000-0000-000000000011", + "memory_key": "owned", + } + ], + audits=[], + documents=[], + ) + asyncio.run( + gate._stage( + SimpleNamespace(fetchval=AsyncMock(return_value=0)), object(), block + ) + ) + return { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, + } + finally: + calls.append("native_returned") + + monkeypatch.setattr(_ch, "run_connected", native) + return output, calls + + +@pytest.mark.parametrize("cause_state", ["57014", "08006", "secret-query-state", True]) +def test_shared_sanitizer_keeps_only_closed_pg_values(cause_state): + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + from server.core.pg import PostgresUnavailableError + + cause = OSError(104, "secret-trigger-diagnostic-payload") + cause.sqlstate = cause_state + error = PostgresUnavailableError("secret-trigger-diagnostic-payload", outcome_unknown=True) + error.__cause__ = cause + metadata = native_failure_metadata(error) + assert metadata["driver"]["outcome_unknown"] is True + entry = metadata["driver"]["causes"][0] + assert entry["error_type"] == "ConnectionResetError" and entry["errno"] == 104 + if type(cause_state) is str and cause_state in {"57014", "08006"}: + assert entry["sqlstate"] == cause_state + else: + assert "sqlstate" not in entry + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + assert len(json.dumps(metadata)) < 4096 + + +def test_shared_sanitizer_preserves_existing_transport_shape_and_cycle_bound(): + from clickhouse_connect.driver.exceptions import OperationalError + + from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata + + error = OperationalError("secret-trigger-diagnostic-payload", code=159) + cause = ConnectionRefusedError(111, "secret-trigger-diagnostic-payload") + error.__cause__ = cause + cause.__cause__ = error + metadata = native_failure_metadata(error) + assert metadata["driver"] == { + "code": 159, + "causes": [{"error_type": "ConnectionRefusedError", "errno": 111}], + } + assert "secret-trigger-diagnostic-payload" not in json.dumps(metadata) + + +def test_diagnostic_error_cannot_replace_actual_stage_invalid(monkeypatch, tmp_path, capsys): + output, calls = _main_case(monkeypatch, tmp_path, "query_cancel") + + def failed(*_args, **_kwargs): + raise RuntimeError("secret-trigger-diagnostic-payload") + + monkeypatch.setattr(gate, "native_failure_metadata", failed) + assert gate.main() == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "INVALID" and verdict["error_type"] == "PostgresUnavailableError" + assert verdict["phase"] == "native_driver" and verdict["reason"] == "operation refused" + assert "failure" not in verdict + assert calls == ["wire", "stage", "native_returned"] + invalid = json.loads(output.read_text().splitlines()[-1]) + assert invalid["kind"] == "invalid" and "failure" not in invalid + assert "secret-trigger-diagnostic-payload" not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "fault", + [ + "none", + "root_sqlstate", + "cause_sqlstate", + "boolean_sqlstate", + "outcome_unknown", + "site_file", + "site_line", + "site_function", + "site_extra", + "driver_extra", + ], +) +def test_native_reader_validates_new_pg_fields_without_replacing_first_error( + keeper_config_case, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = detail() + driver = metadata["driver"] + driver.update(sqlstate="57014", outcome_unknown=True) + cause = driver["causes"][0] + cause.update( + sqlstate="08006", site={"file": "native.py", "line": 702, "function": "connections"} + ) + if fault == "root_sqlstate": + driver["sqlstate"] = "secret-query" + elif fault == "cause_sqlstate": + cause["sqlstate"] = "secret-query" + elif fault == "boolean_sqlstate": + driver["sqlstate"] = True + elif fault == "outcome_unknown": + driver["outcome_unknown"] = 1 + elif fault == "site_file": + cause["site"]["file"] = "/secret/path/native.py" + elif fault == "site_line": + cause["site"]["line"] = True + elif fault == "site_function": + cause["site"]["function"] = "secret-payload" + elif fault == "site_extra": + cause["site"]["message"] = "secret-payload" + elif fault == "driver_extra": + driver["message"] = "secret-payload" + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused("original-command-failure") + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (metadata if fault == "none" else None) From e9bd5b10c291a55872c395a41dfcf3b0a61db47e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:16:38 +0000 Subject: [PATCH 147/181] Retirement census distinguishes method calls from local cleanup Signed-off-by: zlareb1 --- tests/unit/test_cell_retirement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index f3e7593fd3..48517fb11e 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -420,7 +420,7 @@ def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census() commands.add(relative) if name == "participant_command": plain.add(relative) - if name == "stopped": + if name == "stopped" and isinstance(func, ast.Attribute): stopped.add(relative) assert constructors == {"core/retirement.py"} assert writers == {"insights/app/provider_barrier.py"} From 79943b75065eeabd7c03f15b08b9feeba3544081 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 03:50:30 +0000 Subject: [PATCH 148/181] Worker parents retire verified stopped children A child can exit before completing its own retirement. The parent records retirement only after its captured child identity is physically gone and every tagged query has ended, within the existing stop or restart deadline. Signed-off-by: zlareb1 --- server/insights/app/provider_barrier.py | 14 +- .../insights/store/provider_participants.py | 12 +- server/workers/child.py | 74 +++++- tests/unit/test_workers_child_retirement.py | 233 ++++++++++++++++++ 4 files changed, 326 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_workers_child_retirement.py diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 794b457649..26009456e3 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -319,19 +319,25 @@ async def wrapped(app: T) -> AsyncIterator[None]: return wrapped -async def retire_participant(participant: UUID, *, ch: Any = None) -> str: +async def retire_participant( + participant: UUID, *, ch: Any = None, expected_proof: str | None = None +) -> str: """`retired`, `absent` (no row), or `live` (not proved stopped). Retirement needs the recorded identity proved absent in the same host, boot and PID namespace, with tagged queries known zero. Unknown evidence blocks; heartbeat expiry alone proves nothing. """ - row = next( - (r for r in await participants(ch=ch) if str(r["participant_id"]) == str(participant)), - None, + rows = ( + await participants(ch=ch) + if expected_proof is None + else await participants(ch=ch, host_proof=expected_proof) ) + row = next((r for r in rows if str(r["participant_id"]) == str(participant)), None) if row is None: return "absent" + if expected_proof is not None and (len(rows) != 1 or row.get("host_proof") != expected_proof): + return "live" if str(participant) == str(process_identity()): return "live" if not proves_stopped(str(row.get("host_proof") or "")): diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 567aa5e676..e03c5b22bf 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -83,9 +83,17 @@ def running_sql(*, inserts: bool = False) -> str: _PRUNE_SQL = f"ALTER TABLE {TABLE} DELETE WHERE participant_id IN {{ids:Array(UUID)}}" -async def participants(*, ch: _CHClient | None = None) -> list[dict[str, Any]]: +async def participants( + *, ch: _CHClient | None = None, host_proof: str | None = None +) -> list[dict[str, Any]]: with sequential_reads(): - rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + if host_proof is None: + rows = await (ch or _default_client()).query_all(PARTICIPANTS_SQL) + else: + rows = await (ch or _default_client()).query_all( + PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2", + {"proof": host_proof}, + ) return [ r for r in rows diff --git a/server/workers/child.py b/server/workers/child.py index 0d782ca1c4..730ab2478d 100644 --- a/server/workers/child.py +++ b/server/workers/child.py @@ -28,6 +28,8 @@ import sys import time from collections.abc import Callable, Mapping +from pathlib import Path +from uuid import UUID from server.core.errors import error_kind from server.settings import settings @@ -43,6 +45,8 @@ # The longest a stop waits for the child before killing it, whatever the # deadline: the lifespan's shutdown drains share 40 s. STOP_TIMEOUT_S = 20.0 +# Not a setting: proof work fits within the existing stop or restart deadline. +_REAP_RETIRE_S = 1.0 RESTART_DELAY_S = 5.0 # A child that exits sooner than this after it started failed to start; after # MAX_FAILED_STARTS in a row this process gives the lease up. @@ -61,6 +65,8 @@ class ChildWorkers: def __init__(self, lease: leader.SupervisorLease) -> None: self._lease = lease self._proc: asyncio.subprocess.Process | None = None + self._child_proof: str | None = None + self._child_parent_proof: str | None = None self._watch: asyncio.Task[None] | None = None self._publish: asyncio.Task[None] | None = None self._write_fd: int | None = None @@ -120,10 +126,13 @@ async def _keep_running(self) -> None: # The outcome reaches `/v1/health` (no token) as the lease's reason. log.warning("workers child could not start: %s: %s", type(e).__name__, e) outcome = f"could not start ({error_kind(e)})" + restart_at = time.monotonic() + RESTART_DELAY_S else: assert self._proc is not None self.last_exit = await self._proc.wait() + restart_at = time.monotonic() + RESTART_DELAY_S self._close_pipe() + await self._retire_child(self._proc, restart_at) outcome = f"exited (code {self.last_exit})" if not self._lease.fresh(): return # the lease loop stops the workers, and starts them once it is @@ -139,7 +148,7 @@ async def _keep_running(self) -> None: return log.error("workers child %s; restarting in %.0f s", outcome, RESTART_DELAY_S) self.restarts += 1 - await asyncio.sleep(RESTART_DELAY_S) + await asyncio.sleep(max(0.0, restart_at - time.monotonic())) async def _spawn(self) -> None: """Start the child. On an OSError the child's pipe ends are closed here and @@ -163,12 +172,72 @@ async def _spawn(self) -> None: finally: for fd in child_fds: os.close(fd) + self._capture_child_proof() asyncio.get_running_loop().add_reader(self._alive_fd, self._read_beats) log.info("workers child started: pid %s", self._proc.pid) self._publish = asyncio.get_running_loop().create_task( self._publish_loop(), name="loom-workers-child-lease" ) + def _capture_child_proof(self) -> None: + """Capture the owned child's birth while its PID still names our direct child.""" + from server.core import provider_process + + self._child_proof = None + self._child_parent_proof = None + proc = self._proc + if proc is None or proc.returncode is not None or proc.pid <= 1 or proc.pid == os.getpid(): + return + try: + parent_proof = provider_process.host_proof() + parent = parent_proof.split("|") + root = Path(f"/proc/{proc.pid}") + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + if ( + proc.returncode is not None + or len(parent) != 8 + or parent[4] != str(os.getpid()) + or int(stat[1]) != os.getpid() + or str(os.stat(root / "ns/pid").st_ino) != parent[3] + ): + return + parent[4:6] = [str(proc.pid), stat[19]] + self._child_proof = provider_process.certified("|".join(parent)) + self._child_parent_proof = parent_proof + except (OSError, ValueError, IndexError): + return # An unknown birth never authorizes post-reap retirement. + + async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> None: + from server.core import provider_process + from server.insights.app.provider_barrier import retire_participant + from server.insights.store.provider_participants import participants + + proof = self._child_proof + if proc is not self._proc or proc.returncode is None or proof is None: + return + captured = proof.split("|") + parent_proof = provider_process.host_proof() + here = parent_proof.split("|") + if ( + parent_proof != self._child_parent_proof + or len(captured) != 9 + or not provider_process.has_certificate(proof) + or len(here) != 8 + or captured[:4] != here[:4] + or captured[4] != str(proc.pid) + or proc.pid == os.getpid() + or end <= time.monotonic() + ): + return + try: + async with asyncio.timeout(min(_REAP_RETIRE_S, max(0.0, end - time.monotonic()))): + rows = await participants(host_proof=proof) + if len(rows) != 1 or rows[0].get("host_proof") != proof: + return + await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=proof) + except Exception: # noqa: BLE001 — unproved retirement must not replace child cleanup + log.warning("reaped workers child retirement remains unproved") + def _send_deadline(self) -> None: """Send the lease deadline; it moves only while the lease is fresh.""" if self._lease.fresh(): @@ -225,6 +294,9 @@ async def _reap(self, kill_at: float) -> None: with contextlib.suppress(ProcessLookupError): proc.kill() await proc.wait() + if proc is not None: + await proc.wait() + await self._retire_child(proc, kill_at) finally: if proc is not None and proc.returncode is None: with contextlib.suppress(ProcessLookupError): diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py new file mode 100644 index 0000000000..08fc6c8bb3 --- /dev/null +++ b/tests/unit/test_workers_child_retirement.py @@ -0,0 +1,233 @@ +"""The reaped worker child's original process identity gates retirement.""" + +import asyncio +import os +import signal +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.core import provider_process +from server.insights.app import provider_barrier as barrier +from server.insights.store import provider_participants as store +from server.workers import child + + +@pytest.mark.asyncio +async def test_actual_reap_certifies_its_exact_uncertified_child(monkeypatch): + workers, proc, proof, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + evidence = record.await_args.args[0] + assert evidence.participant == UUID(int=17) + assert evidence.host_proof == proof and evidence.after == 9 + barrier.running_queries.assert_awaited_once_with(UUID(int=17), ch=None) + assert proc.signals == [signal.SIGTERM] + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_actual_reap_does_not_rewrite_an_already_certified_child(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch, fault="already_retired") + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "duplicate", + "wrong_pid", + "reused_pid_proof", + "different_namespace", + "own_uuid", + "alive", + "queries_running", + "queries_unknown", + "query_error", + "birth_unknown", + "birth_other_parent", + "birth_wrong_namespace", + "parent_changed", + ], +) +@pytest.mark.asyncio +async def test_reap_retirement_refuses_unknown_or_unsettled_identity(monkeypatch, fault): + workers, proc, _, record, publish = await _owned(monkeypatch, fault=fault) + try: + await workers._reap(time.monotonic() + 10) + assert proc.returncode == -signal.SIGTERM + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_a_live_owned_child_cannot_be_certified_before_its_wait(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._retire_child(proc, time.monotonic() + 10) + assert proc.returncode is None and proc.waits == 0 + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exhausted_original_stop_budget_refuses_retirement(monkeypatch): + workers, proc, _, record, publish = await _owned(monkeypatch) + try: + await workers._reap(time.monotonic() - 1) + assert proc.returncode is not None + assert not record.await_count + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_exact_proof_participant_read_is_bounded_and_parameterized(monkeypatch): + client = SimpleNamespace(query_all=AsyncMock(return_value=[])) + proof = "host|boot|container|77|123|901|||cell-retirement-v1" + assert await store.participants(ch=client, host_proof=proof) == [] + sql, params = client.query_all.await_args.args + assert sql == store.PARTICIPANTS_SQL + " AND host_proof = {proof:String} LIMIT 2" + assert params == {"proof": proof} + assert proof not in sql + + +async def _owned(monkeypatch, *, fault=None): + parent = os.getpid() + pid = parent + 100000 + here = f"host|boot|container|77|{parent}|900||" + proof = f"host|boot|container|77|{pid}|901|||cell-retirement-v1" + rows = [{"participant_id": UUID(int=17), "host_proof": proof, "state": "drained", "version": 9}] + if fault in {"missing", "already_retired"}: + rows = [] + elif fault == "duplicate": + rows *= 2 + elif fault in {"wrong_pid", "reused_pid_proof", "different_namespace"}: + parts = proof.split("|") + parts[{"wrong_pid": 4, "reused_pid_proof": 5, "different_namespace": 3}[fault]] = "999" + rows[0]["host_proof"] = "|".join(parts) + elif fault == "own_uuid": + rows[0]["participant_id"] = UUID(int=18) + participants = AsyncMock(return_value=rows) + monkeypatch.setattr(store, "participants", participants) + monkeypatch.setattr(barrier, "participants", participants) + monkeypatch.setattr(barrier, "process_identity", lambda: UUID(int=18)) + monkeypatch.setattr(barrier, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "proves_stopped", lambda _: fault != "alive") + monkeypatch.setattr(provider_process, "host_proof", lambda: here) + queries = AsyncMock( + return_value=None if fault == "queries_unknown" else int(fault == "queries_running") + ) + if fault == "query_error": + queries.side_effect = RuntimeError("unavailable") + monkeypatch.setattr(barrier, "running_queries", queries) + record = AsyncMock() + monkeypatch.setattr(barrier, "record_retirement", record) + original_read, original_stat = Path.read_text, os.stat + fields = ["S", str(parent)] + ["0"] * 18 + fields[19] = "901" + if fault == "birth_other_parent": + fields[1] = str(parent + 1) + + def read(path, *args, **kwargs): + if path == Path(f"/proc/{pid}/stat"): + if fault == "birth_unknown": + raise PermissionError("uninspectable") + return f"{pid} (python) " + " ".join(fields) + return original_read(path, *args, **kwargs) + + def stat(path, *args, **kwargs): + if str(path) == f"/proc/{pid}/ns/pid": + return SimpleNamespace(st_ino=78 if fault == "birth_wrong_namespace" else 77) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(os, "stat", stat) + + class Process: + returncode = None + waits = 0 + + def __init__(self): + self.pid = pid + self.signals = [] + + def send_signal(self, sig): + self.signals.append(sig) + + def kill(self): + self.returncode = -signal.SIGKILL + + async def wait(self): + self.waits += 1 + self.returncode = -signal.SIGTERM if self.returncode is None else self.returncode + return self.returncode + + proc = Process() + monkeypatch.setattr(child.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc)) + lease = SimpleNamespace(fresh=lambda: True, remaining_s=lambda: 10) + workers = child.ChildWorkers(lease) + await workers._spawn() + if fault == "parent_changed": + monkeypatch.setattr( + provider_process, "host_proof", lambda: f"host|boot|container|77|{parent + 1}|902||" + ) + return workers, proc, proof, record, workers._publish + + +@pytest.mark.asyncio +async def test_naturally_exited_owned_child_is_retired_within_restart_delay(monkeypatch): + workers, _, proof, record, publish = await _owned(monkeypatch) + clock = [time.monotonic()] + sleeps = [] + + class StopLoop(Exception): + pass + + async def record_with_elapsed_proof(_, *, ch=None): + clock[0] += 0.4 + + async def stop_after_restart_delay(delay): + sleeps.append(delay) + raise StopLoop + + record.side_effect = record_with_elapsed_proof + monkeypatch.setattr(workers, "_spawn", AsyncMock()) + monkeypatch.setattr(child, "time", SimpleNamespace(monotonic=lambda: clock[0])) + monkeypatch.setattr(child.asyncio, "sleep", stop_after_restart_delay) + try: + with pytest.raises(StopLoop): + await workers._keep_running() + assert record.await_count == 1 + assert record.await_args.args[0].host_proof == proof + assert sleeps == [pytest.approx(child.RESTART_DELAY_S - 0.4)] + assert workers.restarts == 1 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) From ba5579054a1e2f9ff8c82f122192f11f21252c3d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:18:12 +0000 Subject: [PATCH 149/181] Worker child retirement follows the application boundary Wire exact-proof retirement from the serving composition root and expose its bounded proof timeout through settings. Keep the original child identity, query-settlement and cleanup deadlines. Signed-off-by: zlareb1 --- docs/reference/configuration.md | 1 + loom.example.yaml | 4 +++ schemas/loom-settings.schema.json | 6 +++++ server/insights/app/provider_barrier.py | 8 ++++++ server/main.py | 4 +-- server/settings/catalog.py | 3 +++ server/workers/child.py | 27 ++++++++++----------- tests/unit/snapshots/settings_defaults.json | 1 + tests/unit/test_workers_child.py | 25 ++++++++++--------- tests/unit/test_workers_child_retirement.py | 2 +- 10 files changed, 53 insertions(+), 28 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2c5ba04af8..e4b0045384 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1756,6 +1756,7 @@ Process shape, deployment, event stream and cost budgets. | `LOOM_SUPERVISOR_LEASE_TTL_S` | `int` | `30` | | auto mode: seconds a lease holder that stops renewing keeps the workers before another process takes them over (failover time). At least 24: below that a renewal blocked for its 2 s timeout leaves no retry before the holder must give up (server/workers/leader.py has the timings). | | `LOOM_KEEPER_SWEEP_INTERVAL_SECONDS` | `float` | `3600.0` | | Seconds between passes of the always-on sweep that deletes the KeeperMap keys (leases, claims, graph slots) a stopped process left behind. | | `LOOM_WORKERS_PROCESS` | `'child' \| 'inline'` | `'child'` | | auto mode: where the lease holder runs the workers. child = a separate process it starts and stops with the lease, so the passes do not share the event loop that serves requests; inline = in the serving process. | +| `LOOM_WORKERS_CHILD_RETIREMENT_TIMEOUT_S` | `float` | `1.0` | | At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired. | | `LOOM_SHUTDOWN_READY_DELAY_S` | `float` | `15.0` | | Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so the load balancer stops routing to it before its port closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. | | `LOOM_NATIVE_COMMAND_DRAIN_TIMEOUT_S` | `float` | `15.0` | | Seconds a native CLI command allows its producers, deferred writes and participant retirement to finish. Expiry leaves the participant unretired. | | `LOOM_BOOT_WARMUP_TIMEOUT_S` | `float` | `30.0` | | Seconds boot waits for the first load of the tracker registry and the runtime overrides, and the first publish of the dashboard bundle, before it serves; readiness stays false until the first two load. | diff --git a/loom.example.yaml b/loom.example.yaml index 26b894641c..d3670a9b4f 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -4938,6 +4938,10 @@ settings: # serves requests; inline = in the serving process. [restart] # workers_process: "child" + # At most this many seconds to certify a reaped worker child; also bounded by the + # remaining stop or restart delay. Expiry leaves its participant unretired. [restart] + # workers_child_retirement_timeout_s: 1.0 + # Seconds a process keeps serving after SIGTERM while /v1/health/ready answers 503, so # the load balancer stops routing to it before its port closes. Size it to the LB's # probe interval × failure threshold. 0 = stop at once. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 8f40e96e0d..33ec1f54d0 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -7078,6 +7078,12 @@ "type": "number", "description": "A pass with a failed leg on a worker whose interval is longer than this retries after this long, doubling with each further such pass, up to the interval: a weekly pass must not leave one namespace's transient failure standing for a week." }, + "workers_child_retirement_timeout_s": { + "default": 1.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "At most this many seconds to certify a reaped worker child; also bounded by the remaining stop or restart delay. Expiry leaves its participant unretired." + }, "workers_process": { "default": "child", "enum": [ diff --git a/server/insights/app/provider_barrier.py b/server/insights/app/provider_barrier.py index 26009456e3..8399587f0c 100644 --- a/server/insights/app/provider_barrier.py +++ b/server/insights/app/provider_barrier.py @@ -319,6 +319,14 @@ async def wrapped(app: T) -> AsyncIterator[None]: return wrapped +async def retire_reaped_process(host_proof: str) -> None: + """Retire only the one recorded process matching its owner's captured birth.""" + rows = await participants(host_proof=host_proof) + if len(rows) != 1 or rows[0].get("host_proof") != host_proof: + return + await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=host_proof) + + async def retire_participant( participant: UUID, *, ch: Any = None, expected_proof: str | None = None ) -> str: diff --git a/server/main.py b/server/main.py index 79f7da23ee..a522035bae 100644 --- a/server/main.py +++ b/server/main.py @@ -228,7 +228,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await boot.require_storage_authority(ch) # The deployment's cell state, before the first heartbeat reports this process # serving and before the lease is decided on it. - from server.insights.app.provider_barrier import register_process + from server.insights.app.provider_barrier import register_process, retire_reaped_process await register_process() for task in await boot.start_cell_gate(): @@ -273,7 +273,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: # the background workers, the holder of the ClickHouse lease, in a child # process of its own unless LOOM_WORKERS_PROCESS=inline. lease = await _leader.decide_supervisor(settings.workers_mode()) - supervisor = _workers_child.worker_runner(lease) + supervisor = _workers_child.worker_runner(lease, retire_child=retire_reaped_process) _leader.set_current(lease) # Apply per-table retention overrides from LOOM_RETENTION_*_DAYS, if set — # from the deployment's leader only, and only where the TTL differs. diff --git a/server/settings/catalog.py b/server/settings/catalog.py index a9aceefc9f..b5da3d7607 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5176,6 +5176,9 @@ class RuntimeSettings(BaseModel): # process it starts and stops with the lease, so the passes do not share # the event loop that serves requests; inline = in the serving process. workers_process: Literal["child", "inline"] = "child" + # At most this many seconds to certify a reaped worker child; also bounded by + # the remaining stop or restart delay. Expiry leaves its participant unretired. + workers_child_retirement_timeout_s: float = Field(default=1.0, gt=0, allow_inf_nan=False) # Seconds a process keeps serving after SIGTERM while /v1/health/ready # answers 503, so the load balancer stops routing to it before its port # closes. Size it to the LB's probe interval × failure threshold. 0 = stop at once. diff --git a/server/workers/child.py b/server/workers/child.py index 730ab2478d..6a24b8510f 100644 --- a/server/workers/child.py +++ b/server/workers/child.py @@ -27,9 +27,8 @@ import signal import sys import time -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path -from uuid import UUID from server.core.errors import error_kind from server.settings import settings @@ -45,8 +44,6 @@ # The longest a stop waits for the child before killing it, whatever the # deadline: the lifespan's shutdown drains share 40 s. STOP_TIMEOUT_S = 20.0 -# Not a setting: proof work fits within the existing stop or restart deadline. -_REAP_RETIRE_S = 1.0 RESTART_DELAY_S = 5.0 # A child that exits sooner than this after it started failed to start; after # MAX_FAILED_STARTS in a row this process gives the lease up. @@ -62,8 +59,11 @@ class ChildWorkers: """The supervisor's start/stop/status surface, backed by a child process.""" - def __init__(self, lease: leader.SupervisorLease) -> None: + def __init__( + self, lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] + ) -> None: self._lease = lease + self._retire_callback = retire_child self._proc: asyncio.subprocess.Process | None = None self._child_proof: str | None = None self._child_parent_proof: str | None = None @@ -209,8 +209,6 @@ def _capture_child_proof(self) -> None: async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> None: from server.core import provider_process - from server.insights.app.provider_barrier import retire_participant - from server.insights.store.provider_participants import participants proof = self._child_proof if proc is not self._proc or proc.returncode is None or proof is None: @@ -230,11 +228,10 @@ async def _retire_child(self, proc: asyncio.subprocess.Process, end: float) -> N ): return try: - async with asyncio.timeout(min(_REAP_RETIRE_S, max(0.0, end - time.monotonic()))): - rows = await participants(host_proof=proof) - if len(rows) != 1 or rows[0].get("host_proof") != proof: - return - await retire_participant(UUID(str(rows[0]["participant_id"])), expected_proof=proof) + async with asyncio.timeout( + min(settings.workers_child_retirement_timeout_s, max(0.0, end - time.monotonic())) + ): + await self._retire_callback(proof) except Exception: # noqa: BLE001 — unproved retirement must not replace child cleanup log.warning("reaped workers child retirement remains unproved") @@ -330,12 +327,14 @@ def child_env(parent: Mapping[str, str]) -> dict[str, str]: _RUNNER: WorkerSupervisor | ChildWorkers | None = None -def worker_runner(lease: leader.SupervisorLease) -> WorkerSupervisor | ChildWorkers: +def worker_runner( + lease: leader.SupervisorLease, *, retire_child: Callable[[str], Awaitable[None]] +) -> WorkerSupervisor | ChildWorkers: """What runs the workers here: a child process for an `auto` lease holder (`LOOM_WORKERS_PROCESS=child`), else the in-process supervisor.""" global _RUNNER if lease.mode == "auto" and settings.workers_process == "child": - _RUNNER = ChildWorkers(lease) + _RUNNER = ChildWorkers(lease, retire_child=retire_child) else: _RUNNER = get_supervisor() return _RUNNER diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 804775722c..fd7142a9bf 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -1258,6 +1258,7 @@ "worker_backoff_cap_s": 60.0, "worker_ledger_rows": true, "worker_partial_retry_s": 3600.0, + "workers_child_retirement_timeout_s": 1.0, "workers_process": "child", "workload_exchanges_per_agent": 300, "workload_exchanges_per_trust": 50000 diff --git a/tests/unit/test_workers_child.py b/tests/unit/test_workers_child.py index 1f3c1854a6..57269fd3b2 100644 --- a/tests/unit/test_workers_child.py +++ b/tests/unit/test_workers_child.py @@ -12,6 +12,7 @@ import threading import time from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -83,7 +84,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa monkeypatch.setattr( child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", _STUB, str(fd), str(out)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: out.exists() and bool(out.read_text())) deadline = float(out.read_text()) @@ -96,7 +97,7 @@ async def test_parent_sends_the_deadline_and_stops_the_child(monkeypatch, tmp_pa def test_the_deadline_stops_moving_once_the_lease_is_not_fresh() -> None: lease = _Lease(remaining=12.0) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers._send_deadline() sent = workers._deadline lease.remaining = 0.0 # lost @@ -111,7 +112,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d ) -> None: ready = _stubborn(monkeypatch, tmp_path) lease = _Lease(remaining=0.3) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) lease.remaining = 0.0 # the lease is lost: the deadline stays where it was @@ -124,7 +125,7 @@ async def test_a_child_that_ignores_sigterm_is_killed_at_the_deadline_plus_the_d @pytest.mark.asyncio async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: ready = _stubborn(monkeypatch, tmp_path) - workers = child.ChildWorkers(_Lease(remaining=30.0)) + workers = child.ChildWorkers(_Lease(remaining=30.0), retire_child=AsyncMock()) workers.start_all() await _until(ready.exists) proc = workers._proc @@ -139,7 +140,7 @@ async def test_a_cancelled_stop_kills_the_child(monkeypatch, tmp_path) -> None: async def test_a_child_that_exits_unasked_is_started_again(monkeypatch) -> None: monkeypatch.setattr(child, "RESTART_DELAY_S", 0.01) monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "pass"]) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers.restarts >= 2) await workers.stop_all() @@ -153,7 +154,7 @@ async def test_a_child_that_keeps_failing_to_start_gives_the_lease_up(monkeypatc child, "child_argv", lambda fd, _alive_fd: [sys.executable, "-c", "exit(3)"] ) lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and workers.restarts == 2 @@ -173,7 +174,7 @@ async def test_a_child_that_cannot_be_created_gives_the_lease_up_and_leaks_no_fd monkeypatch.setattr(child, "child_argv", lambda fd, _alive_fd: [missing]) fds = _open_fds() lease = _Lease() - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=AsyncMock()) workers.start_all() await _until(lambda: bool(lease.given_up)) assert workers.failed_starts == 3 and "could not start" in lease.given_up[0] @@ -204,7 +205,7 @@ async def test_a_child_whose_loop_hangs_is_not_alive(monkeypatch) -> None: monkeypatch.setattr( child, "child_argv", lambda _fd, alive_fd: [sys.executable, "-c", _HANGS, str(alive_fd)] ) - workers = child.ChildWorkers(_Lease()) + workers = child.ChildWorkers(_Lease(), retire_child=AsyncMock()) workers.start_all() await _until(lambda: workers._proc is not None) started = workers._beat_at @@ -304,12 +305,14 @@ def test_an_auto_holder_runs_its_workers_in_a_child(monkeypatch) -> None: lease = SupervisorLease("auto") monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(lease), child.ChildWorkers) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), child.ChildWorkers) assert child.current_runner() is child._RUNNER monkeypatch.setattr(settings, "workers_process", "inline") - assert isinstance(child.worker_runner(lease), WorkerSupervisor) + assert isinstance(child.worker_runner(lease, retire_child=AsyncMock()), WorkerSupervisor) monkeypatch.setattr(settings, "workers_process", "child") - assert isinstance(child.worker_runner(SupervisorLease("on")), WorkerSupervisor) + assert isinstance( + child.worker_runner(SupervisorLease("on"), retire_child=AsyncMock()), WorkerSupervisor + ) def test_the_child_runs_its_own_entry_point_not_the_serving_app() -> None: diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py index 08fc6c8bb3..d41972c7dd 100644 --- a/tests/unit/test_workers_child_retirement.py +++ b/tests/unit/test_workers_child_retirement.py @@ -191,7 +191,7 @@ async def wait(self): proc = Process() monkeypatch.setattr(child.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc)) lease = SimpleNamespace(fresh=lambda: True, remaining_s=lambda: 10) - workers = child.ChildWorkers(lease) + workers = child.ChildWorkers(lease, retire_child=barrier.retire_reaped_process) await workers._spawn() if fault == "parent_changed": monkeypatch.setattr( From 23bd673479d4db9525e682d0df56374c7d24815d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:45:57 +0000 Subject: [PATCH 150/181] Verify captured worker subtrees in native recovery checks Capture exact background-child births before controlled crash and recreation. Keep certified retirement and settled queries mandatory for every captured participant, and report bounded role counts on failure. Signed-off-by: zlareb1 --- scripts/gates/docker_provider_data.py | 126 +++++++++++- .../test_docker_provider_owned_subtree.py | 187 ++++++++++++++++++ tests/unit/test_docker_provider_smoke.py | 3 +- 3 files changed, 306 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_docker_provider_owned_subtree.py diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 7f8611f850..059e40e3d8 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -9,12 +9,17 @@ import time from multiprocessing import spawn from pathlib import Path +from typing import TYPE_CHECKING, Any from uuid import UUID from server.core import provider_process from server.core.ch import get_client from server.insights.app.provider_command import model_command from server.settings import settings +from server.workers.child import child_argv + +if TYPE_CHECKING: + from server.core.repo_base import _CHClient SUPERVISOR = [ "-m", @@ -149,21 +154,93 @@ def _workers(rows: list[dict], parent: dict) -> list[dict]: return sorted(workers, key=lambda row: row["host_proof"]) -async def _worker_rows(client) -> list[dict]: - return await client.query_all( +def _proof(process: dict[str, Any]) -> str: + parts = provider_process.host_proof().split("|") + assert len(parts) == 8 and all(parts[:4]), "Incomplete fixture process scope" + parts[4:6] = [str(process["pid"]), process["start"]] + return provider_process.certified("|".join(parts)) + + +async def _proof_rows(client: "_CHClient", proofs: list[str]) -> list[dict[str, Any]]: + assert 1 <= len(proofs) <= 4 and len(set(proofs)) == len(proofs) + rows = await client.query_all( "SELECT toString(participant_id) AS participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE state != 'retired'" + "FROM loom._provider_participants FINAL WHERE state != 'retired' " + "AND host_proof IN {proofs:Array(String)} LIMIT 5", + {"proofs": proofs}, ) + assert len(rows) <= len(proofs), "Duplicate captured process registrations" + return rows + + +async def _worker_rows(client) -> list[dict]: + return await _proof_rows(client, [_proof(p) for p in _kernel_workers().values()]) + + +def _background_children( + roots: list[dict[str, Any]], parent: dict[str, Any] +) -> dict[str, list[dict[str, Any]]]: + found = {} + for root in roots: + web = _worker_process(root["host_proof"], parent) + children = [] + for value in Path(f"/proc/{web['pid']}/task/{web['pid']}/children").read_text().split(): + process = _process(int(value)) + command = process["command"] + if "server.worker_main" not in command: + continue + assert process["parent"] == web["pid"], "Background child parent changed" + assert process["namespace"] == web["namespace"], "Background child namespace changed" + assert process["executable"] == web["executable"], "Background child executable changed" + assert len(command) == 7, "Background child command mismatch" + fds = [int(command[4]), int(command[6])] + assert min(fds) >= 0 and command == child_argv(*fds), ( + "Background child command mismatch" + ) + children.append(process) + assert len(children) <= 1, "More than one background child per web worker" + assert _worker_process(root["host_proof"], parent) == web, ( + "Web worker changed during child census" + ) + found[root["participant_id"]] = children + return found async def workers(client) -> list[dict]: parent = _supervisor() result = _workers(await _worker_rows(client), parent) assert len(result) == 2, "Worker registration census is incomplete" + children = _background_children(result, parent) + proofs = [_proof(p) for values in children.values() for p in values] + rows = await _proof_rows(client, proofs) if proofs else [] + for root in result: + root["children"] = [] + for process in children[root["participant_id"]]: + matches = [row for row in rows if row["host_proof"] == _proof(process)] + if not matches: + raise _WorkersPending("Background child exact registration is pending") + assert len(matches) == 1, "Background child registration is ambiguous" + row = matches[0] + assert provider_process.has_certificate(row["host_proof"]), ( + "Background child certificate invalid" + ) + root["children"].append( + {key: row[key] for key in ("participant_id", "host_proof", "version")} + ) + assert _background_children(result, parent) == children, ( + "Background child identities changed during census" + ) + assert len({row["participant_id"] for row in _captured(result)}) == 2 + len(proofs) assert _supervisor() == parent, "Fixture supervisor changed during registration census" return result +def _captured(roots: list[dict[str, Any]]) -> list[dict[str, Any]]: + assert 1 <= len(roots) <= 2 + assert all(type(root["children"]) is list and len(root["children"]) <= 1 for root in roots) + return [row for root in roots for row in (root, *root["children"])] + + def _kill_worker(worker: dict, parent: dict) -> None: proof = worker["host_proof"].split("|") pid = int(proof[4]) @@ -180,9 +257,10 @@ def _kill_worker(worker: dict, parent: dict) -> None: async def _retirement_rows(client, captured: list[dict]) -> list[dict]: + assert 1 <= len(captured) <= 4 rows = await client.query_all( "SELECT participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", {"ids": [row["participant_id"] for row in captured]}, ) return [{**row, "participant_id": str(row["participant_id"])} for row in rows] @@ -232,11 +310,38 @@ def result(reason: str) -> bool: return result("proved") +async def _subtree_diagnostic( + client: "_CHClient", roots: list[dict[str, Any]], *, proved: bool +) -> None: + """Print closed role counts, preserving the original retirement failure.""" + try: + roles = {"web": roots, "background": [c for r in roots for c in r["children"]]} + summary = {} + for role, captured in roles.items(): + counts = { + "captured": len(captured), + "retirement_proved": 0, + "death_proved": 0, + "query_zero_proved": 0, + "unproved": 0, + } + for row in captured: + counts["death_proved"] += int(provider_process.proves_stopped(row["host_proof"])) + accepted = proved or (bool(roles["background"]) and await retired(client, [row])) + counts["retirement_proved"] += int(accepted) + counts["query_zero_proved"] += int(accepted) + counts["unproved"] += int(not accepted) + summary[role] = counts + print("Provider subtree retirement diagnostic: " + json.dumps(summary, sort_keys=True)) + except BaseException: # noqa: BLE001 — diagnostics must preserve the original retirement failure + print("Provider subtree retirement diagnostic unavailable") + + async def crash_worker(client) -> None: parent = _supervisor() before = await workers(client) victim = before[0] - for worker in before: + for worker in _captured(before): rows = await _retirement_rows(client, [worker]) identity_matches = len(rows) == 1 and rows[0]["participant_id"] == worker["participant_id"] if not identity_matches: @@ -255,7 +360,7 @@ async def crash_worker(client) -> None: assert _supervisor() == parent stopped = provider_process.proves_stopped(victim["host_proof"]) observation = {"death": "proved" if stopped else "unproved", "retirement": "not_checked"} - if stopped and await retired(client, [victim], observation=observation): + if stopped and await retired(client, _captured([victim]), observation=observation): try: after = _workers(await _worker_rows(client), parent) except _WorkersPending: @@ -270,10 +375,12 @@ async def crash_worker(client) -> None: victim_absent=victim["participant_id"] not in new, ) if len(new) == 2 and len(old & new) == 1 and victim["participant_id"] not in new: + await _subtree_diagnostic(client, [victim], proved=True) return await asyncio.sleep(1) if observation["retirement"] == "row_count": observation.update(await _native_retirement_observation(client, [victim])) + await _subtree_diagnostic(client, [victim], proved=False) raise AssertionError( "Crashed worker retirement or supervised replacement was not proved: " + json.dumps(observation, sort_keys=True) @@ -418,9 +525,10 @@ async def main() -> None: elif sys.argv[1] == "crash-worker": await crash_worker(client) elif sys.argv[1] == "retired": - assert await retired(client, json.loads(sys.argv[2])), ( - "Recreated workers did not retire" - ) + captured = json.loads(sys.argv[2]) + accepted = await retired(client, _captured(captured)) + await _subtree_diagnostic(client, captured, proved=accepted) + assert accepted, "Recreated workers did not retire" else: await verify(sys.argv[1], int(sys.argv[2])) finally: diff --git a/tests/unit/test_docker_provider_owned_subtree.py b/tests/unit/test_docker_provider_owned_subtree.py new file mode 100644 index 0000000000..1eefcbe195 --- /dev/null +++ b/tests/unit/test_docker_provider_owned_subtree.py @@ -0,0 +1,187 @@ +"""The native fixture captures and checks its exact controlled process subtree.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +import scripts.gates.docker_provider_data as data +from server.insights.store import provider_participants +from server.workers.child import child_argv +from tests.unit.test_docker_provider_smoke import _process_fixture + + +def _fixture(monkeypatch, *, background=True, fault=None): + parent, processes, roots = _process_fixture(monkeypatch) + children = {1: "8 9", 8: "18" if background else "", 9: ""} + descendant = { + **processes[8], + "pid": 18, + "parent": 8, + "start": "180", + "command": child_argv(4, 8), + } + processes[18] = descendant + child = { + "participant_id": "background-participant", + "version": 5, + "state": "running", + "host_proof": "h|b|c|1|18|180|||cell-retirement-v1", + } + registrations = [*roots, *([child] if background else [])] + if fault == "root-reused": + processes[8] = {**processes[8], "start": "81"} + elif fault == "root-parent": + processes[8] = {**processes[8], "parent": 2} + elif fault == "child-reused": + processes[18] = {**descendant, "start": "181"} + elif fault == "child-parent": + processes[18] = {**descendant, "parent": 9} + elif fault == "child-namespace": + processes[18] = {**descendant, "namespace": 2} + elif fault == "child-command": + processes[18] = {**descendant, "command": [*descendant["command"], "--unknown"]} + elif fault == "two-background": + processes[19] = {**descendant, "pid": 19, "start": "190"} + children[8] = "18 19" + elif fault == "missing-child": + registrations = roots + elif fault == "duplicate-child": + registrations.append({**child, "participant_id": "ambiguous-background"}) + + def read(path): + parts = str(path).split("/") + assert parts[1] == "proc" and parts[3] == "task" and parts[2] == parts[4] + assert parts[5] == "children" + return children[int(parts[2])] + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data, "_supervisor", lambda: parent) + + async def query(sql, parameters=None): + selected = registrations + if parameters and "proofs" in parameters: + selected = [r for r in selected if r["host_proof"] in parameters["proofs"]] + if ( + fault == "child-changed-after-query" + and parameters + and child["host_proof"] in parameters.get("proofs", []) + ): + processes[18] = {**descendant, "start": "181"} + return selected + + client = SimpleNamespace(query_all=AsyncMock(side_effect=query)) + return roots, child, client + + +@pytest.mark.asyncio +async def test_actual_workers_capture_the_exact_background_child(monkeypatch): + roots, child, client = _fixture(monkeypatch) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert [c["participant_id"] for r in captured for c in r.get("children", [])] == [ + child["participant_id"] + ] + assert len(captured) == 2 + + +@pytest.mark.asyncio +async def test_actual_workers_keep_the_same_two_web_roots_without_a_background_child(monkeypatch): + roots, _, client = _fixture(monkeypatch, background=False) + captured = await data.workers(client) + assert [r["participant_id"] for r in captured] == [r["participant_id"] for r in roots] + assert not any(r.get("children", []) for r in captured) + assert len(captured) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["root-reused", "root-parent"]) +async def test_actual_workers_refuse_changed_web_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + "child-reused", + "child-parent", + "child-namespace", + "child-command", + "two-background", + "missing-child", + "duplicate-child", + "child-changed-after-query", + ], +) +async def test_actual_workers_refuse_uncertain_background_identity(monkeypatch, fault): + _, _, client = _fixture(monkeypatch, fault=fault) + with pytest.raises(AssertionError): + await data.workers(client) + + +@pytest.mark.asyncio +async def test_actual_recreated_check_refuses_an_unretired_background_child(monkeypatch): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in roots] + [ + {**child, "state": "drained", "version": 6} + ] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + with pytest.raises(AssertionError, match="Recreated workers did not retire"): + await data.main() + client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_actual_recreated_check_reports_only_closed_role_proof_counts(monkeypatch, capsys): + roots, child, _ = _fixture(monkeypatch) + captured = [{**r, "children": [child] if i == 0 else []} for i, r in enumerate(roots)] + observed = [{**r, "state": "retired", "version": 6} for r in [*roots, child]] + + async def query(_sql, parameters): + return [r for r in observed if r["participant_id"] in parameters["ids"]] + + client = SimpleNamespace( + connect=AsyncMock(), disconnect=AsyncMock(), query_all=AsyncMock(side_effect=query) + ) + monkeypatch.setattr(data, "get_client", lambda: client) + monkeypatch.setattr(data.sys, "argv", ["fixture", "retired", json.dumps(captured)]) + monkeypatch.setattr(provider_participants, "running_queries", AsyncMock(return_value=0)) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + await data.main() + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + assert summary == { + "web": { + "captured": 2, + "retirement_proved": 2, + "death_proved": 0, + "query_zero_proved": 2, + "unproved": 0, + }, + "background": { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + }, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output diff --git a/tests/unit/test_docker_provider_smoke.py b/tests/unit/test_docker_provider_smoke.py index 64119f0817..a7c59cf446 100644 --- a/tests/unit/test_docker_provider_smoke.py +++ b/tests/unit/test_docker_provider_smoke.py @@ -253,6 +253,7 @@ def _process_fixture(monkeypatch): rows = [ { "participant_id": str(pid), + "children": [], "version": 5, "state": "running", "host_proof": f"h|b|c|1|{pid}|{pid * 10}|||cell-retirement-v1", @@ -537,7 +538,7 @@ async def test_retirement_reader_preserves_string_identity_from_native_uuid(): assert row["participant_id"] is participant_id assert client.query_all.await_args.args == ( "SELECT participant_id, state, version, host_proof " - "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)}", + "FROM loom._provider_participants FINAL WHERE participant_id IN {ids:Array(UUID)} LIMIT 5", {"ids": [str(participant_id)]}, ) From 74d59a42e8b762441937e9c326fea71e4a889b48 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:05:08 +0000 Subject: [PATCH 151/181] Test child retirement limits and clarify native token minting Cover finite positive retirement budgets and the original remaining allowance. Distinguish an already issued scoped token from native minting after enforcement. Signed-off-by: zlareb1 --- docs/operations/cell-attach.md | 7 ++-- tests/unit/test_workers_child_retirement.py | 40 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 53e176d1cd..451de55f3e 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -115,9 +115,10 @@ and `attaching`, and the cell goes forward: fix the cause and rerun attach. Unse `LOOM_CELL` then gates every process, an outage, not a rollback. A row with `placed > 0` always goes forward. -From `enforced` on, a `--platform-admin` mint that declares a new org and the admin account -routes cannot bring in a new org (`WRONG_CELL`); an ordinary mint scoped to its org places -the org on its first request. A direct call of the router's operator place route, or a live +From `enforced` on, native minting requires the org to be held by this cell, with or +without `--platform-admin`. Admin account routes cannot bring in a new org (`WRONG_CELL`). +An already issued ordinary token scoped to its org places that org on its first request. +A direct call of the router's operator place route, or a live placement with the cell's credential, cannot read the cell's Postgres and is accepted once the directory mode is `enforced`; the cell's credential is the servers' alone. diff --git a/tests/unit/test_workers_child_retirement.py b/tests/unit/test_workers_child_retirement.py index d41972c7dd..bfa1829226 100644 --- a/tests/unit/test_workers_child_retirement.py +++ b/tests/unit/test_workers_child_retirement.py @@ -14,7 +14,9 @@ from server.core import provider_process from server.insights.app import provider_barrier as barrier from server.insights.store import provider_participants as store +from server.settings import Settings from server.workers import child +from tests.unit.clock_fake import ManualClock @pytest.mark.asyncio @@ -231,3 +233,41 @@ async def stop_after_restart_delay(delay): workers._close_pipe() publish.cancel() await asyncio.gather(publish, return_exceptions=True) + + +@pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf"), -float("inf")]) +def test_child_retirement_proof_timeout_requires_a_positive_finite_setting(value): + with pytest.raises(ValueError): + Settings(_env_file=None, workers_child_retirement_timeout_s=value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "remaining", "expected"), + [(0.125, 2.0, 0.125), (1.0, 0.25, 0.25), (2.0, 0.0625, 0.0625)], +) +async def test_child_retirement_setting_caps_only_the_original_remaining_allowance( + monkeypatch, configured, remaining, expected +): + workers, proc, _, record, publish = await _owned(monkeypatch) + clock = ManualClock(t=100.0) + allowances = [] + original_timeout = asyncio.timeout + + def observed_timeout(delay): + allowances.append(delay) + return original_timeout(delay) + + monkeypatch.setattr(child.settings, "workers_child_retirement_timeout_s", configured) + monkeypatch.setattr(child, "time", clock) + monkeypatch.setattr(child.asyncio, "timeout", observed_timeout) + try: + await workers._reap(clock.monotonic() + remaining) + assert proc.returncode == -signal.SIGTERM and proc.waits >= 1 + assert record.await_count == 1 + assert allowances == [expected] + assert clock.monotonic() == 100.0 + finally: + workers._close_pipe() + publish.cancel() + await asyncio.gather(publish, return_exceptions=True) From 9d6d5861f1554917a1b3628427da4025b3ead016 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:29:24 +0000 Subject: [PATCH 152/181] Prove exact exit of single-thread zombie participants Signed-off-by: zlareb1 --- docs/architecture/cells.md | 7 ++ scripts/gates/docker_provider_data.py | 27 +++++ server/core/provider_process.py | 16 ++- tests/unit/test_provider_host_proof.py | 62 ++++++++++ tests/unit/test_provider_zombie_exit.py | 152 ++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_provider_zombie_exit.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 7038121a79..45b319c1d8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -36,6 +36,13 @@ setup; its optional analyst-row work opens a separate verified, enrolled scope and obtains fresh acceptance. That bootstrap exception admits no tenant or token work. +Local physical-death evidence requires the same host, kernel boot and PID namespace. +It accepts an absent PID, a different process birth, or the captured birth in a +single fresh kernel process record showing a zombie with exactly one thread. A +zombie leader with other threads is unproved. Known-zero tagged queries remain a +separate requirement before retirement; kernel process exit does not settle a +query still executing in a store. + The capability identifies installed code; it does not prove that every older process or unsafe retirement client has disappeared. diff --git a/scripts/gates/docker_provider_data.py b/scripts/gates/docker_provider_data.py index 059e40e3d8..175b4eac3c 100644 --- a/scripts/gates/docker_provider_data.py +++ b/scripts/gates/docker_provider_data.py @@ -324,8 +324,35 @@ async def _subtree_diagnostic( "death_proved": 0, "query_zero_proved": 0, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 0, + "kernel_missing": 0, + "kernel_uninspectable": 0, } for row in captured: + kernel = "kernel_uninspectable" + proof = row["host_proof"].split("|") + here = provider_process.host_proof().split("|") + if len(proof) in {8, 9} and proof[:4] == here[:4] and proof[4].isdecimal(): + try: + raw = await asyncio.to_thread(Path(f"/proc/{int(proof[4])}/stat").read_text) + stat = raw.rsplit(")", 1)[1].split() + if stat[19] == proof[5]: + threads = int(stat[17]) + if stat[0] == "Z" and threads >= 1: + kernel = ( + "kernel_zombie_single_thread" + if threads == 1 + else "kernel_zombie_multi_thread" + ) + elif threads >= 1: + kernel = "kernel_present_non_zombie" + except FileNotFoundError: + kernel = "kernel_missing" + except (OSError, IndexError, ValueError): + pass # An unreadable identity does not establish a kernel state. + counts[kernel] += 1 counts["death_proved"] += int(provider_process.proves_stopped(row["host_proof"])) accepted = proved or (bool(roles["background"]) and await retired(client, [row])) counts["retirement_proved"] += int(accepted) diff --git a/server/core/provider_process.py b/server/core/provider_process.py index a2c6ad6328..aa88ffe6fc 100644 --- a/server/core/provider_process.py +++ b/server/core/provider_process.py @@ -96,8 +96,8 @@ def host_proof() -> str: def proves_stopped(proof: str) -> bool: """Whether the process `proof` names has stopped, judged from this one: same host - name and kernel boot, and either its pid is free or runs another process than - the one that started then (same pid namespace). Another namespace or API + name and kernel boot, and its pid is free, runs a different birth, or is a + single-thread zombie of that birth (same pid namespace). Another namespace or API object proves nothing. Sharing a host name or container identity alone does not prove death; an uninspectable process is not proven stopped.""" parts = proof.split("|") @@ -116,7 +116,7 @@ def proves_stopped(proof: str) -> bool: return False if pidns != my_pidns: return False # namespace identity alone says nothing about old processes - # Only positive evidence: no such process, or another one under its pid. + # Only positive evidence: absence, a different birth, or a fully exited group. # A pid this process may not inspect (hidepid, a permission error) proves nothing. try: os.kill(int(pid), 0) @@ -125,8 +125,14 @@ def proves_stopped(proof: str) -> bool: except (OSError, ValueError): return False try: - return _start_ticks(pid) != start - except (OSError, IndexError): + if _start_ticks(pid) != start: + return True + with open(f"/proc/{pid}/stat", encoding="ascii") as f: + stat = f.read().rsplit(")", 1)[1].split() + # A zombie leader can still have live siblings. Bind all three fields to + # one fresh record: only a single-thread zombie cannot resume this birth. + return stat[19] == start and stat[0] == "Z" and stat[17] == "1" + except (OSError, IndexError, ValueError): return False diff --git a/tests/unit/test_provider_host_proof.py b/tests/unit/test_provider_host_proof.py index 0e70602dd7..295f8dc03f 100644 --- a/tests/unit/test_provider_host_proof.py +++ b/tests/unit/test_provider_host_proof.py @@ -160,3 +160,65 @@ async def test_this_host_retires_its_dead_participants_once_their_queries_ended( assert proof.participant == dead and proof.after == 5 assert proof.host_proof == pp.certified(gone) assert await barrier.retire_dead_here(now=100.0 + barrier.HERE_S / 2) == 0 # throttled + + +def _fresh_process_stat( + monkeypatch: pytest.MonkeyPatch, text: str, error: type[Exception] | None = None +) -> str: + """Hold the first birth check steady and supply the subsequent kernel record.""" + from io import StringIO + + monkeypatch.setattr(pp.socket, "gethostname", lambda: "host") + monkeypatch.setattr(pp, "_boot_and_pidns", lambda: ("boot", "77")) + monkeypatch.setattr(pp.os, "kill", lambda pid, sig: None) + monkeypatch.setattr(pp, "_start_ticks", lambda pid: "900") + + def fresh_open(path: str, *args: Any, **kwargs: Any) -> StringIO: + assert path == "/proc/424242/stat" + if error is not None: + raise error("uninspectable fresh process record") + return StringIO(text) + + monkeypatch.setattr(pp, "open", fresh_open, raising=False) + return "host|boot|container|77|424242|900||" + + +def _process_stat_record(state: str = "Z", threads: str = "1", start: str = "900") -> str: + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "424242 (owned child) " + " ".join(fields) + + +def test_an_exact_single_thread_zombie_record_proves_exit(monkeypatch: pytest.MonkeyPatch) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record()) + assert pp.proves_stopped(proof) + + +@pytest.mark.parametrize( + "record", + [ + _process_stat_record(threads="0"), + _process_stat_record(threads="2"), + _process_stat_record(threads="-1"), + _process_stat_record(threads="unknown"), + _process_stat_record(threads="1.0"), + *[_process_stat_record(state=state) for state in ("S", "D", "R", "T", "t", "X")], + _process_stat_record(start="901"), + _process_stat_record(start="unknown"), + "424242 (owned child) Z", + "unparseable kernel record", + ], +) +def test_fresh_kernel_record_must_prove_the_same_single_thread_exit( + monkeypatch: pytest.MonkeyPatch, record: str +) -> None: + proof = _fresh_process_stat(monkeypatch, record) + assert not pp.proves_stopped(proof) + + +@pytest.mark.parametrize("error", [PermissionError, OSError, IndexError]) +def test_unreadable_fresh_kernel_record_never_proves_exit( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + proof = _fresh_process_stat(monkeypatch, _process_stat_record(), error) + assert not pp.proves_stopped(proof) diff --git a/tests/unit/test_provider_zombie_exit.py b/tests/unit/test_provider_zombie_exit.py new file mode 100644 index 0000000000..501f3f56a1 --- /dev/null +++ b/tests/unit/test_provider_zombie_exit.py @@ -0,0 +1,152 @@ +"""Exact single-thread process exit is provable before its parent collects it.""" + +import ctypes +import os +import select +import signal +import threading +import time +from pathlib import Path + +import pytest + +from server.core import provider_process + +pytestmark = pytest.mark.skipif(not provider_process.host_proof(), reason="needs /proc") + + +def _stat(pid: int) -> tuple[str, int, str, int]: + parts = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return parts[0], int(parts[1]), parts[19], int(parts[17]) + + +def _wait_owned(pid: int, start: str) -> int: + """Collect only this fork's child, within a finite cleanup allowance.""" + for force in (False, True): + end = time.monotonic() + 2.0 + if force: + try: + _, parent, now_start, _ = _stat(pid) + except FileNotFoundError: + parent, now_start = None, None + if parent == os.getpid() and now_start == start: + os.kill(pid, signal.SIGKILL) + while time.monotonic() < end: + found, status = os.waitpid(pid, os.WNOHANG) + if found == pid: + return status + time.sleep(0.005) + raise AssertionError("Owned witness child did not settle within cleanup allowance") + + +def test_an_exact_single_thread_zombie_is_proven_stopped_before_wait() -> None: + """A captured child has exited, even while its parent retains the wait status.""" + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.read(read_fd, 1) + os._exit(0) + os.close(read_fd) + start = None + released = False + try: + state, parent, start, threads = _stat(pid) + assert state != "Z" and parent == os.getpid() and threads == 1 + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert not provider_process.proves_stopped(exact), "Live child must refuse" + wrong_namespace = proof.copy() + wrong_namespace[3] = str(int(proof[3]) + 1) + assert not provider_process.proves_stopped("|".join(wrong_namespace)) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z": + break + time.sleep(0.005) + assert state == "Z" and threads == 1, ( + "Controlled child must reach single-thread Z before any waitpid" + ) + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact), "After waitpid, absence proves death" + + +def test_a_zombie_leader_is_refused_until_its_last_live_thread_exits() -> None: + """Z alone is insufficient while another thread of that exact process runs.""" + read_fd, write_fd = os.pipe() + ready_read, ready_write = os.pipe() + pid = os.fork() + if pid == 0: + os.close(write_fd) + os.close(ready_read) + + def live_thread() -> None: + os.write(ready_write, b"r") + os.read(read_fd, 1) + os._exit(0) + + threading.Thread(target=live_thread).start() + libc = ctypes.CDLL(None) + libc.pthread_exit.argtypes = [ctypes.c_void_p] + libc.pthread_exit.restype = None + libc.pthread_exit(None) + os._exit(99) # pthread_exit must not return. + os.close(read_fd) + os.close(ready_write) + start = None + released = False + try: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + proof = provider_process.host_proof().split("|") + assert len(proof) == 8 and all(proof[i] for i in (0, 1, 3)) + proof[4:6] = [str(pid), start] + exact = "|".join(proof) + assert select.select([ready_read], [], [], 2.0)[0] == [ready_read] + assert os.read(ready_read, 1) == b"r" + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads > 1: + break + time.sleep(0.005) + assert state == "Z" and threads > 1, "Live-thread leader-Z witness was not established" + assert not provider_process.proves_stopped(exact) + os.write(write_fd, b"x") + released = True + end = time.monotonic() + 2.0 + while time.monotonic() < end: + state, parent, observed_start, threads = _stat(pid) + assert parent == os.getpid() and observed_start == start + if state == "Z" and threads == 1: + break + time.sleep(0.005) + assert state == "Z" and threads == 1, "Controlled group exit must reach Z1 before waitpid" + assert provider_process.proves_stopped(exact) + finally: + if not released: + os.write(write_fd, b"x") + os.close(write_fd) + os.close(ready_read) + if start is None: + _, parent, start, _ = _stat(pid) + assert parent == os.getpid() + status = _wait_owned(pid, start) + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + assert provider_process.proves_stopped(exact) From da95f2de6c79b0622f13040f8f605b0537b5bac4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:52:35 +0000 Subject: [PATCH 153/181] Match subtree diagnostic controls to kernel observations Model the captured process stat records and assert all closed kernel counts. Keep identity and secrecy controls and cover missing, unreadable and reused process observations. Signed-off-by: zlareb1 --- .../test_docker_provider_owned_subtree.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/unit/test_docker_provider_owned_subtree.py b/tests/unit/test_docker_provider_owned_subtree.py index 1eefcbe195..05b3fcc7ef 100644 --- a/tests/unit/test_docker_provider_owned_subtree.py +++ b/tests/unit/test_docker_provider_owned_subtree.py @@ -52,6 +52,11 @@ def _fixture(monkeypatch, *, background=True, fault=None): def read(path): parts = str(path).split("/") + if len(parts) == 4 and parts[1] == "proc" and parts[3] == "stat": + process = processes[int(parts[2])] + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = "S", "1", process["start"] + return f"{process['pid']} (fixture worker) " + " ".join(fields) assert parts[1] == "proc" and parts[3] == "task" and parts[2] == parts[4] assert parts[5] == "children" return children[int(parts[2])] @@ -174,6 +179,11 @@ async def query(_sql, parameters): "death_proved": 0, "query_zero_proved": 2, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 2, + "kernel_missing": 0, + "kernel_uninspectable": 0, }, "background": { "captured": 1, @@ -181,7 +191,71 @@ async def query(_sql, parameters): "death_proved": 0, "query_zero_proved": 1, "unproved": 0, + "kernel_zombie_single_thread": 0, + "kernel_zombie_multi_thread": 0, + "kernel_present_non_zombie": 1, + "kernel_missing": 0, + "kernel_uninspectable": 0, }, } assert all(row["host_proof"] not in output for row in [*roots, child]) assert child["participant_id"] not in output + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("state", "threads", "start", "error", "expected"), + [ + ("Z", "1", "180", None, "kernel_zombie_single_thread"), + ("Z", "2", "180", None, "kernel_zombie_multi_thread"), + ("S", "1", "180", None, "kernel_present_non_zombie"), + ("S", "1", "180", FileNotFoundError, "kernel_missing"), + ("S", "1", "180", PermissionError, "kernel_uninspectable"), + ("Z", "1", "181", None, "kernel_uninspectable"), + ], +) +async def test_subtree_kernel_counts_observe_only_the_captured_identity( + monkeypatch, capsys, state, threads, start, error, expected +): + roots, child, _ = _fixture(monkeypatch) + captured = [{**roots[0], "children": [child]}] + fixture_path = data.Path + + def read(path): + if path != "/proc/18/stat": + return fixture_path(path).read_text() + if error is not None: + raise error("kernel record unavailable") + fields = ["0"] * 20 + fields[0], fields[17], fields[19] = state, threads, start + return "18 (fixture child) " + " ".join(fields) + + monkeypatch.setattr(data, "Path", lambda path: SimpleNamespace(read_text=lambda: read(path))) + monkeypatch.setattr(data.provider_process, "proves_stopped", lambda _: False) + client = SimpleNamespace(query_all=AsyncMock(side_effect=AssertionError("no query expected"))) + await data._subtree_diagnostic(client, captured, proved=True) + prefix = "Provider subtree retirement diagnostic: " + output = capsys.readouterr().out + assert output.startswith(prefix) + summary = json.loads(output.removeprefix(prefix)) + common = { + "captured": 1, + "retirement_proved": 1, + "death_proved": 0, + "query_zero_proved": 1, + "unproved": 0, + } + kernel_keys = { + "kernel_zombie_single_thread", + "kernel_zombie_multi_thread", + "kernel_present_non_zombie", + "kernel_missing", + "kernel_uninspectable", + } + assert summary == { + "web": {**common, **{key: int(key == "kernel_present_non_zombie") for key in kernel_keys}}, + "background": {**common, **{key: int(key == expected) for key in kernel_keys}}, + } + assert all(row["host_proof"] not in output for row in [*roots, child]) + assert child["participant_id"] not in output + client.query_all.assert_not_awaited() From a1ebaad9602c9868e1995c18ec880742c59125e0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:12:05 +0000 Subject: [PATCH 154/181] Distinguish store guard failures and reap interrupted test children Keep the existing store checks and failure envelope while giving each refused observation its own source site. Protect the native test child with the same kill and wait ownership as neighboring tests. Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 8 +-- .../test_directory_control_cleanup.py | 9 +++- .../unit/test_cell_trigger_resource_census.py | 34 +++++++++++++ tests/unit/test_directory_control_cleanup.py | 50 +++++++++++++++++++ 4 files changed, 95 insertions(+), 6 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 27c280092c..4634ea5081 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -782,11 +782,11 @@ def sample(self, *, admit: bool = False) -> None: boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() for store in self.spec["stores"].values(): identity = self.owned.identity(store["host_pid"]) + need(identity is not None, "resource store birth changed") + need(identity[1] == store["start_ticks"], "resource store birth changed") + need(boot == store["boot_id"], "resource store birth changed") need( - identity is not None - and identity[1] == store["start_ticks"] - and boot == store["boot_id"] - and os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], + os.stat(f"/proc/{store['host_pid']}").st_uid == store["uid"], "resource store birth changed", ) roots = [self.owned.pid] + [s["host_pid"] for s in self.spec["stores"].values()] diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index 2df016414a..c07ce79ac8 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -300,8 +300,13 @@ async def work(conn): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await asyncio.wait_for(process.communicate(), 10) - assert process.returncode == 0, (stdout.decode(), stderr.decode()) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), 10) + assert process.returncode == 0, (stdout.decode(), stderr.decode()) + finally: + if process.returncode is None: + process.kill() + await process.wait() @pytest.mark.parametrize("kind", ["ordinary", "cancel", "exit", "keyboard"]) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index 31a10c22cd..fb3cf02058 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -240,3 +240,37 @@ def stat(path, *args, **kwargs): assert isinstance(owner.first, fixture.Refused) assert owner.sampled == 7.0 assert owned.round == 1 + + +def test_store_identity_refusals_keep_distinct_failure_sites(census, monkeypatch): + owner, owned, _ = census + original_identity, original_stat = owned.identify, fixture.os.stat + sites = [] + for fault in ("missing", "start", "boot", "uid"): + owner.failures.clear() + owner.sampled = 7.0 + owner.spec["stores"]["pg"]["boot_id"] = "changed" if fault == "boot" else "boot" + owned.identify = lambda pid, fault=fault: ( + None + if pid == 200 and fault == "missing" + else (1, "changed", "S") + if pid == 200 and fault == "start" + else original_identity(pid) + ) + + def stat(path, *args, fault=fault, **kwargs): + if str(path) == "/proc/200" and fault == "uid": + return SimpleNamespace(st_uid=fixture.os.geteuid() + 1) + return original_stat(path, *args, **kwargs) + + monkeypatch.setattr(fixture.os, "stat", stat) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert str(owner.first) == "resource store birth changed" + assert owner.sampled == 7.0 + record = owner.failure_summary()["records"][0] + assert record["primary"] is True and record["phase"] == "resource" + assert record["site"]["file"] == "fixture.py" + assert record["site"]["function"] == "sample" + sites.append(record["site"]["line"]) + assert len(set(sites)) == 4 diff --git a/tests/unit/test_directory_control_cleanup.py b/tests/unit/test_directory_control_cleanup.py index be6fee8329..c8dbe8ed0e 100644 --- a/tests/unit/test_directory_control_cleanup.py +++ b/tests/unit/test_directory_control_cleanup.py @@ -309,3 +309,53 @@ async def observed(): assert result is first assert began.is_set() conn.terminate.assert_called_once_with() + + +@pytest.mark.parametrize("ending", ["timeout", "cancel", "healthy"]) +async def test_integration_fatal_child_reaps_its_process_after_interruption(monkeypatch, ending): + from tests.integration import test_directory_control_cleanup as native + + original = { + "timeout": TimeoutError("owned child communication timed out"), + "cancel": asyncio.CancelledError("owned child communication cancelled"), + "healthy": None, + }[ending] + observer = Mock(fetchval=AsyncMock(return_value="owned_fake")) + process = Mock(returncode=None) + + async def communicate(): + if original is not None: + raise original + process.returncode = 0 + return b"", b"" + + async def wait(): + process.returncode = -9 + return process.returncode + + process.communicate = AsyncMock(side_effect=communicate) + process.wait = AsyncMock(side_effect=wait) + create = AsyncMock(return_value=process) + monkeypatch.setenv("LOOM_PG_DSN", "postgresql://u:test-only@127.0.0.1:15432/admin") + monkeypatch.setattr(native.asyncio, "create_subprocess_exec", create) + if original is None: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + else: + with pytest.raises(type(original)) as caught: + await native.test_directory_fatal_child_is_inert_until_owned_cleanup( + observer, "work", "exit" + ) + assert caught.value is original + observer.fetchval.assert_awaited_once_with("SELECT current_database()") + create.assert_awaited_once() + process.communicate.assert_awaited_once() + if original is None: + assert process.returncode == 0 + process.kill.assert_not_called() + process.wait.assert_not_awaited() + else: + process.kill.assert_called_once_with() + process.wait.assert_awaited_once_with() + assert process.returncode == -9 From 3d86ef4534de27a094c269027a41af1f1fae6668 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:30:04 +0000 Subject: [PATCH 155/181] Wait for the published ClickHouse HTTP listener The initialization server can answer native queries before its final HTTP listener starts. Probe the published endpoint through the existing bounded creator command. Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 45 ++++++----- tests/unit/test_cell_trigger_ci_creator.py | 38 +++++++++ ...test_cell_trigger_ci_creator_http_ready.py | 79 +++++++++++++++++++ 3 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_cell_trigger_ci_creator_http_ready.py diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 52701f103b..d43d81d4f0 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -20,6 +20,26 @@ GIB = 1024**3 SUPPORT = Path(__file__).resolve().parent / "testsupport/cell_trigger" +_CLICKHOUSE_READY = """import http.client +import sys + +connection = http.client.HTTPConnection("127.0.0.1", int(sys.argv[1]), timeout=2) +ready = False +try: + connection.request( + "POST", "/", body="SELECT 1", + headers={"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"}, + ) + response = connection.getresponse() + ready = response.status == 200 and response.read(3) == b"1\\n" +except (OSError, http.client.HTTPException): + pass +finally: + connection.close() +if ready: + print("1") +""" + def require(ok: object, message: str) -> None: if not ok: @@ -246,6 +266,12 @@ def wait(self, phase: str, probe, seconds: int = 120) -> None: require(time.monotonic() < end, "Owned store readiness deadline") time.sleep(1) + def _clickhouse_ready(self, port: int) -> bool: + return ( + self.command(sys.executable, "-I", "-S", "-c", _CLICKHOUSE_READY, str(port), timeout=2) + == "1" + ) + def failure_diagnostics(self) -> dict: observations = [] statuses = {"created", "running", "paused", "restarting", "removing", "exited", "dead"} @@ -517,24 +543,7 @@ def start(self, ch_image: str) -> tuple[dict, dict, dict, list[int]]: == "sync" ), ) - self.wait( - "clickhouse_query", - lambda: ( - self.command( - "docker", - "exec", - ch["id"], - "clickhouse-client", - "--user", - "loom", - "--password", - "loomdev", - "--query", - "SELECT 1", - ) - == "1" - ), - ) + self.wait("clickhouse_query", lambda: self._clickhouse_ready(ch_port)) return ch, pg, standby, selected def observation(self, item: dict, role: str) -> dict: diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index e1e77569c5..730f265968 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -219,6 +219,9 @@ def __init__(self): self.tcp_attempts = 0 self.tcp_ready = False + def _clickhouse_ready(self, port): + return True + def create(self, role, args): return {"id": role} @@ -790,3 +793,38 @@ def diagnostic(*args, **kwargs): assert error is original and original.native is None else: pytest.fail("command refusal was swallowed") + + +def test_clickhouse_waits_for_published_http_after_temporary_native_ready(tmp_path, monkeypatch): + from scripts.gates import cell_trigger_acceptance as creator + + monkeypatch.setattr(creator, "ports", lambda count: list(range(18000, 18000 + count))) + monkeypatch.setattr(creator.time, "sleep", lambda _: None) + + class InitializingClickHouse(Stores): + def __init__(self): + super().__init__(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + self.published_attempts = 0 + self.native_attempts = 0 + + def create(self, role, args): + return {"id": role} + + def command(self, *argv, timeout=120, cleanup=False): + if "clickhouse-client" in argv: + self.native_attempts += 1 + return "1" # The temporary loopback-only server already accepts native queries. + if "pg_isready" in argv: + return "127.0.0.1:5432 - accepting connections" + return "sync" if "SELECT sync_state" in argv[-1] else "network-id" + + def _clickhouse_ready(self, port): + assert port == 18000 + self.published_attempts += 1 + return self.published_attempts == 2 + + stores = InitializingClickHouse() + stores.start("test-image") + assert stores.published_attempts == 2 + assert stores.native_attempts == 0 + assert stores.phase == "clickhouse_query" diff --git a/tests/unit/test_cell_trigger_ci_creator_http_ready.py b/tests/unit/test_cell_trigger_ci_creator_http_ready.py new file mode 100644 index 0000000000..8bad85a598 --- /dev/null +++ b/tests/unit/test_cell_trigger_ci_creator_http_ready.py @@ -0,0 +1,79 @@ +"""The fixed creator probe accepts only the published HTTP query response.""" + +import http.client +import sys + +import pytest + +from scripts.gates import cell_trigger_acceptance as creator + + +@pytest.mark.parametrize( + ("status", "body", "failure", "expected"), + [ + (200, b"1\n", None, True), + (200, b"1\nx", None, False), + (200, b"1", None, False), + (200, b"", None, False), + (500, b"1\n", None, False), + (200, b"1\n", "disconnect", False), + (200, b"1\n", "socket", False), + ], +) +def test_fixed_http_probe_checks_response_and_closes_connection( + monkeypatch, capsys, status, body, failure, expected +): + calls = [] + + class Response: + def __init__(self): + self.status = status + + def read(self, amount): + assert amount == 3 + calls.append("read") + return body + + class Connection: + def request(self, method, path, *, body, headers): + assert (method, path, body) == ("POST", "/", "SELECT 1") + assert headers == {"X-ClickHouse-User": "loom", "X-ClickHouse-Key": "loomdev"} + calls.append("request") + if failure == "socket": + raise OSError("not listening") + + def getresponse(self): + calls.append("response") + if failure == "disconnect": + raise http.client.RemoteDisconnected("bootstrap stopped") + return Response() + + def close(self): + calls.append("close") + + def connect(host, port, *, timeout): + assert (host, port, timeout) == ("127.0.0.1", 18000, 2) + return Connection() + + monkeypatch.setattr(http.client, "HTTPConnection", connect) + monkeypatch.setattr(sys, "argv", ["probe", "18000"]) + exec(creator._CLICKHOUSE_READY, {}) + assert capsys.readouterr().out == ("1\n" if expected else "") + assert calls[-1] == "close" + assert calls.count("close") == 1 + assert calls.count("read") == (1 if status == 200 and failure is None else 0) + + +def test_readiness_uses_existing_bounded_creator_command(tmp_path): + class RecordingStores(creator.Stores): + def command(self, *argv, timeout=120, cleanup=False): + self.recorded = (argv, timeout, cleanup) + return "1" + + stores = RecordingStores(tmp_path, tmp_path, "fixture", {"run_id": "123"}) + assert stores._clickhouse_ready(18000) + assert stores.recorded == ( + (sys.executable, "-I", "-S", "-c", creator._CLICKHOUSE_READY, "18000"), + 2, + False, + ) From 00ceb04854511d41425fe9fbd6bb545f8be46726 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:50:17 +0000 Subject: [PATCH 156/181] Wait for the complete single-thread exit witness A zombie leader can still report multiple threads. Keep polling within the existing allowance until the complete asserted exit state is observed. Signed-off-by: zlareb1 --- tests/unit/test_provider_zombie_exit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_provider_zombie_exit.py b/tests/unit/test_provider_zombie_exit.py index 501f3f56a1..db6bd2868f 100644 --- a/tests/unit/test_provider_zombie_exit.py +++ b/tests/unit/test_provider_zombie_exit.py @@ -67,7 +67,7 @@ def test_an_exact_single_thread_zombie_is_proven_stopped_before_wait() -> None: while time.monotonic() < end: state, parent, observed_start, threads = _stat(pid) assert parent == os.getpid() and observed_start == start - if state == "Z": + if state == "Z" and threads == 1: break time.sleep(0.005) assert state == "Z" and threads == 1, ( From 71121962969eb18c46c44a80cfb35c8ca9f42ae4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:00:16 +0000 Subject: [PATCH 157/181] Reap the retirement fixture child after interrupted waits The fixture owns the child through timeout and cancellation. Keep its existing wait budget and always collect that exact process before store cleanup. Signed-off-by: zlareb1 --- tests/integration/test_cell_retirement.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_cell_retirement.py b/tests/integration/test_cell_retirement.py index a769a0728e..8032268818 100644 --- a/tests/integration/test_cell_retirement.py +++ b/tests/integration/test_cell_retirement.py @@ -95,7 +95,12 @@ async def test_certified_retirement_survives_old_beat_and_prunes_only_certified_ ) stopped, legacy = uuid4(), uuid4() process = await asyncio.create_subprocess_exec(sys.executable, "-c", "pass") - await asyncio.wait_for(process.wait(), timeout=10) + try: + await asyncio.wait_for(process.wait(), timeout=10) + finally: + if process.returncode is None: + process.kill() + await process.wait() fields = provider_process.host_proof().split("|") fields[4:6] = [str(process.pid), "1"] identity = "|".join(fields) From 1d752a74c1c7f06addafd428cde6802986348a48 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:08:55 +0000 Subject: [PATCH 158/181] Collect the attach lock fixture's delayed rollback task Cancel and join the owned task before dedicated sessions close on an interrupted control operation. Preserve the contention checks and wait allowances. Signed-off-by: zlareb1 --- tests/integration/test_cell_attach.py | 31 ++++++++++++++++----------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 71a2b41b71..72f6604578 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -647,19 +647,24 @@ async def release() -> None: await tx.rollback() task = asyncio.create_task(release()) - await cell_attach_pg.take_lock(conn, "mode test") - own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") - await cell_attach_pg.commit_enforced( - _owner(conn), - ok, - ok, - cell_attach_pg.Wait(20), - logged.append, - lambda _m: None, - expected_cell="first", - expected_fp=(own["current_fp"], own["next_fp"]), - ) - await task + try: + await cell_attach_pg.take_lock(conn, "mode test") + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + await cell_attach_pg.commit_enforced( + _owner(conn), + ok, + ok, + cell_attach_pg.Wait(20), + logged.append, + lambda _m: None, + expected_cell="first", + expected_fp=(own["current_fp"], own["next_fp"]), + ) + await task + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) assert await _mode() == "enforced" assert logged == ["the mode's lock is busy; retrying"] From efcfc407712f23b35894092730377390a5308d07 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:30:05 +0000 Subject: [PATCH 159/181] Isolate retirement gates between integration scenarios Give each in-process test its own real gate while keeping shutdown permanent within the scenario. A private patch context preserves the fixture through explicit test monkeypatch cleanup. Signed-off-by: zlareb1 --- tests/integration/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e84ef1b5a1..acf7a1b199 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -250,6 +250,16 @@ async def asend(self: httpx.AsyncClient, request: httpx.Request, **kw: Any) -> h mp.undo() +@pytest.fixture(autouse=True) +def isolated_retirement_gate() -> Iterator[None]: + """Each test owns a fresh process gate; shutdown stays permanent within it.""" + from server.core import retirement + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(retirement, "_gate", retirement.RetirementGate()) + yield + + @pytest.fixture(autouse=True) def _code_registry_read() -> Iterator[None]: """In-process code ops answer only once the self-service registry has been read From 96bd15ee4b19036d66301b8d5b5c6ac6dbfb9e8f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:56:14 +0000 Subject: [PATCH 160/181] Trigger acceptance retains bounded staging failure evidence A canceled head-lock query leaves its cancellation mechanism and query plan unknown. Record closed reasons and a bounded fresh planning observation while retaining the original failure and cleanup contract. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 110 ++++++- .../gates/testsupport/cell_trigger/fixture.py | 11 + .../gates/testsupport/cell_trigger/native.py | 10 + ...est_cell_trigger_driver_failure_details.py | 7 +- .../test_cell_trigger_stage_cancellation.py | 291 ++++++++++++++++++ 5 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_cell_trigger_stage_cancellation.py diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 18237c300c..9cf9f5b206 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1453,8 +1453,13 @@ async def _run( async with asyncio.timeout(SETTLE_S): await _settle_api(conn, ch, args, api) block = await _build_block(args, batch) - async with asyncio.timeout(SETTLE_S): - await _stage(conn, ch, block) + stage_started = time.perf_counter_ns() + try: + async with asyncio.timeout(SETTLE_S): + await _stage(conn, ch, block) + except BaseException as error: + await _stage_failure(conn, block, artifact, arm, stage_started, error) + raise rps = await _delivery(args, batch, emit=artifact.emit, arm=arm) async with asyncio.timeout(SETTLE_S): await _verify(conn, ch, args, block) @@ -1512,6 +1517,107 @@ async def _run( raise asyncio.CancelledError +def _stage_plan(value: Any) -> list[dict[str, Any]]: + if type(value) is str: + _require(len(value) <= 65_536, "bounded stage plan bytes") + value = json.loads(value) + _require(type(value) is list and len(value) == 1, "one stage plan") + pending = [(value[0]["Plan"], 0)] + nodes = [] + node_types = { + "Nested Loop", + "Hash Join", + "Merge Join", + "Seq Scan", + "Index Scan", + "Index Only Scan", + "Bitmap Heap Scan", + "Bitmap Index Scan", + "Sort", + "Incremental Sort", + "Function Scan", + "LockRows", + "Hash", + "Materialize", + "Memoize", + "Gather", + "Gather Merge", + "Result", + "Append", + "Merge Append", + "Unique", + "Aggregate", + "Limit", + } + join_types = {"Inner", "Left", "Right", "Full", "Semi", "Anti", "Right Anti"} + while pending: + node, depth = pending.pop() + _require(type(node) is dict and len(nodes) < 32 and depth <= 8, "bounded stage plan") + kind = node.get("Node Type") + out: dict[str, Any] = { + "depth": depth, + "node_type": kind if type(kind) is str and kind in node_types else "other", + } + if "Join Type" in node: + join = node["Join Type"] + out["join_type"] = join if type(join) is str and join in join_types else "other" + for source, target in ( + ("Plan Rows", "plan_rows"), + ("Startup Cost", "startup_cost"), + ("Total Cost", "total_cost"), + ): + number = node.get(source) + _require( + type(number) in {int, float} and math.isfinite(number) and 0 <= number <= 1e18, + "bounded stage plan estimate", + ) + out[target] = number + nodes.append(out) + children = node.get("Plans", []) + _require(type(children) is list and len(children) <= 32, "bounded stage plan children") + pending.extend((child, depth + 1) for child in reversed(children)) + return nodes + + +async def _stage_failure( + conn: Any, block: Block, artifact: Artifact, arm: int, started: int, error: BaseException +) -> None: + try: + artifact.emit( + "stage_error", + arm=arm, + key_count=len(block.rows), + elapsed_ms=max(0, time.perf_counter_ns() - started) / 1_000_000, + ) + except BaseException: # noqa: BLE001 — stage diagnostics cannot replace the work failure. + pass + try: + causes = native_failure_metadata(error).get("driver", {}).get("causes", []) + if not any(cause.get("sqlstate") == "57014" for cause in causes): + return + from server.memory.store import pg_memory + + async with asyncio.timeout(IO_S): + _require(not conn.is_in_transaction(), "idle stage plan observer") + keys = sorted({pg_memory.identity_key(row) for row in block.rows}) + plan = await conn.fetchval( + "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS, + *pg_memory._key_arrays(keys), + ) + # This independent session plans afresh after rollback; it cannot expose + # the failed pooled backend's cached plan or its former wait state. + artifact.emit( + "freshplan_after_failure", + arm=arm, + key_count=len(keys), + observer="independent_connection", + analyze=False, + nodes=_stage_plan(plan), + ) + except BaseException: # noqa: BLE001 — a failed observation preserves the stage's first error. + pass + + async def _cleanup(triggers: Triggers | None, conn: Any, artifact: Artifact) -> None: end = time.monotonic() + RESTORE_S try: diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index 4634ea5081..93c7a6ff44 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -641,6 +641,17 @@ def native_failure_metadata(error: BaseException, *, files: tuple[str, ...] = FI sqlstate = getattr(cause, "sqlstate", None) if type(sqlstate) is str and re.fullmatch(r"[0-9A-Z]{5}", sqlstate): causes[-1]["sqlstate"] = sqlstate + if name == "QueryCanceledError" and sqlstate == "57014": + # Compare fixed server messages; exception text never enters the receipt. + message = getattr(cause, "message", None) + causes[-1]["cancel_reason"] = ( + { + "canceling statement due to statement timeout": "statement_timeout", + "canceling statement due to user request": "user_request", + }.get(message, "unknown") + if type(message) is str + else "unknown" + ) cause = cause.__cause__ if cause.__cause__ is not None else cause.__context__ driver = {"code": code, "causes": causes} sqlstate = getattr(error, "sqlstate", None) diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 8d68c386db..79e58d48a9 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -1535,6 +1535,7 @@ def action( cause, "error_type errno" + (" sqlstate" if "sqlstate" in cause else "") + + (" cancel_reason" if "cancel_reason" in cause else "") + (" site" if "site" in cause else ""), ) need( @@ -1543,6 +1544,15 @@ def action( and re.fullmatch(r"[0-9A-Z]{5}", cause["sqlstate"]), "native Postgres failure cause", ) + if "cancel_reason" in cause: + need( + cause["error_type"] == "QueryCanceledError" + and cause.get("sqlstate") == "57014" + and type(cause["cancel_reason"]) is str + and cause["cancel_reason"] + in {"statement_timeout", "user_request", "unknown"}, + "native Postgres cancellation reason", + ) if "site" in cause: cause_site = cause["site"] closed(cause_site, "file line function") diff --git a/tests/unit/test_cell_trigger_driver_failure_details.py b/tests/unit/test_cell_trigger_driver_failure_details.py index ff41e5e20e..15db973026 100644 --- a/tests/unit/test_cell_trigger_driver_failure_details.py +++ b/tests/unit/test_cell_trigger_driver_failure_details.py @@ -39,7 +39,12 @@ def test_actual_stage_failure_retains_pg_cause_in_invalid_result( expected = ( {"error_type": "ConnectionResetError", "errno": 104} if fault == "connection" - else {"error_type": "QueryCanceledError", "errno": None, "sqlstate": "57014"} + else { + "error_type": "QueryCanceledError", + "errno": None, + "sqlstate": "57014", + "cancel_reason": "unknown", + } ) assert metadata["driver"]["causes"] == [expected] invalid = json.loads(output.read_text().splitlines()[-1]) diff --git a/tests/unit/test_cell_trigger_stage_cancellation.py b/tests/unit/test_cell_trigger_stage_cancellation.py new file mode 100644 index 0000000000..4fcc55148b --- /dev/null +++ b/tests/unit/test_cell_trigger_stage_cancellation.py @@ -0,0 +1,291 @@ +"""Actual staging failures retain bounded cancellation and fresh-plan evidence.""" + +import asyncio +import json +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import asyncpg +import pytest + +from bench import cell_trigger_gate as gate +from scripts.gates.testsupport.cell_trigger.fixture import native_failure_metadata +from server.core import pg +from server.memory.store import pg_memory +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_driver_failure_details import _main_case +from tests.unit.test_cell_trigger_gate_diagnostics import run_driver as run_driver + +SECRET = "secret-stage-SQL-value-condition-credential" +MESSAGES = [ + ("canceling statement due to statement timeout", "statement_timeout"), + ("canceling statement due to user request", "user_request"), + (SECRET, "unknown"), +] +SUMMARY = { + "latency_within_limit": True, + "throughput_within_limit": True, + "p99_delta_ms": 0.0, + "throughput_loss": 0.0, +} + + +def canceled(message): + try: + raise asyncpg.PostgresError.new({"C": "57014", "M": message}) + except asyncpg.QueryCanceledError as cause: + try: + pg._raise_if_unavailable(cause) + except pg.PostgresUnavailableError as error: + return error + raise AssertionError("the real PostgreSQL wrapper must refuse") + + +def plan_node(kind="Hash Join"): + return { + "Node Type": kind, + "Join Type": "Inner", + "Plan Rows": 0, + "Startup Cost": 1.0, + "Total Cost": 2.0, + "Relation Name": SECRET, + "Index Name": SECRET, + "Hash Cond": SECRET, + "Index Cond": SECRET, + "Output": [SECRET], + "SQL": SECRET, + } + + +@pytest.fixture +def stage_case(monkeypatch, tmp_path, run_driver): + def configure(*, fault=None, message=MESSAGES[0][0], plan_fault=None): + artifact_error = OSError(SECRET) if fault == "artifact" else None + cleanup_error = RuntimeError(SECRET) if fault == "explain" else None + run, events = run_driver( + prior=None, cleanup=cleanup_error, output=artifact_error, event="stage_error" + ) + rows = [ + { + "org": "owned", + "namespace": "scratch", + "agent": "bench", + "user_id": "-", + "session_id": "-", + "memory_key": key, + "memory_id": f"00000000-0000-0000-0000-{index:012d}", + } + for index, key in enumerate(("b", "a"), 1) + ] + block = gate.Block(rows=rows, audits=[{}, {}], documents=[]) + first = ( + None + if fault == "healthy" + else ValueError(SECRET) + if fault == "noncancel" + else KeyboardInterrupt(SECRET) + if fault == "fatal" + else canceled(message) + ) + state = SimpleNamespace(first=first, explain=[], commits=[], block=block) + conn = gate.asyncpg.connect.return_value + monkeypatch.setattr(conn, "is_in_transaction", lambda: False, raising=False) + + async def fetchval(sql, *values): + if sql.startswith("EXPLAIN"): + state.explain.append((sql, values)) + assert sql == "EXPLAIN (FORMAT JSON) " + pg_memory._LOCK_HEADS + assert values == ( + ["owned", "owned"], + ["scratch", "scratch"], + ["bench", "bench"], + ["-", "-"], + ["-", "-"], + ["a", "b"], + ) + if fault == "explain": + raise OSError(SECRET) + if plan_fault == "oversized": + return " " * 65_537 + node = plan_node() + if plan_fault == "deep": + for _ in range(9): + node = {**plan_node("Materialize"), "Plans": [node]} + elif plan_fault == "nonfinite": + node["Total Cost"] = float("nan") + else: + node["Plans"] = [plan_node("Index Scan"), plan_node("Function Scan")] + return json.dumps([{"Plan": node, "Private": SECRET}]) + if "fresh" in sql: + raise AssertionError("freshness is an invariant label, never SQL") + if "memory_id=ANY" in sql: + return 0 + assert "targets=7" in sql + return len(rows) + + async def commit(actual_rows, actual_audits, **kwargs): + state.commits.append((actual_rows, actual_audits, kwargs)) + assert actual_rows is block.rows and actual_audits is block.audits + assert kwargs == {"targets": 7, "publisher_lease": False} + if first is not None: + raise first + return SimpleNamespace( + seqs=[("owned", "scratch", index) for index in (1, 2)], + targets=[7, 7], + rows=rows, + audits=block.audits, + ) + + monkeypatch.setattr(conn, "fetchval", fetchval, raising=False) + monkeypatch.setattr(gate, "_arm_order", lambda: (True,)) + monkeypatch.setattr( + gate, + "_memory_set", + AsyncMock(return_value=[SimpleNamespace(measured=True, elapsed_ms=1.0)]), + ) + monkeypatch.setattr(gate, "_settle_api", AsyncMock()) + monkeypatch.setattr(gate, "_build_block", AsyncMock(return_value=block)) + monkeypatch.setattr(gate, "_delivery", AsyncMock(return_value=100.0)) + monkeypatch.setattr(gate, "_verify", AsyncMock()) + monkeypatch.setattr(gate, "_physical", AsyncMock(return_value={})) + monkeypatch.setattr(gate, "_debt", AsyncMock()) + monkeypatch.setattr(gate, "_absent", AsyncMock()) + monkeypatch.setattr(gate, "_statistics", lambda samples: SUMMARY.copy()) + monkeypatch.setattr(pg_memory, "commit_versions", commit) + + async def observe(): + try: + result = await run() + except BaseException as error: # noqa: BLE001 — keep native fatals inside their owner. + result = error + records = [ + json.loads(line) for line in (tmp_path / "raw.jsonl").read_text().splitlines() + ] + assert SECRET not in json.dumps(records) + return result, records + + return run, observe, events, state + + return configure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("message,reason", MESSAGES) +async def test_actual_stage_records_cancellation_before_preserving_original( + stage_case, message, reason +): + _, observe, events, state = stage_case(message=message) + result, records = await observe() + stage = [record for record in records if record["kind"] == "stage_error"] + assert len(stage) == 1 # Baseline fails here after the real _run -> _stage refusal. + assert stage[0]["arm"] == 0 and stage[0]["key_count"] == 2 + assert type(stage[0]["elapsed_ms"]) in {int, float} and stage[0]["elapsed_ms"] >= 0 + assert result is state.first and len(state.commits) == len(state.explain) == 1 + metadata = native_failure_metadata(result) + cause = metadata["driver"]["causes"][0] + assert cause["cancel_reason"] == reason and cause["sqlstate"] == "57014" + assert metadata["driver"]["outcome_unknown"] is False + assert SECRET not in json.dumps(metadata) + fresh = [record for record in records if record["kind"] == "freshplan_after_failure"] + assert len(fresh) == 1 + assert fresh[0]["observer"] == "independent_connection" and fresh[0]["analyze"] is False + assert fresh[0]["key_count"] == 2 + assert fresh[0]["nodes"] == [ + { + "depth": depth, + "node_type": kind, + "join_type": "Inner", + "plan_rows": 0, + "startup_cost": 1.0, + "total_cost": 2.0, + } + for depth, kind in ((0, "Hash Join"), (1, "Index Scan"), (1, "Function Scan")) + ] + assert events.index("stage_error") < events.index("restoration") + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ["noncancel", "fatal", "explain", "artifact"]) +async def test_actual_stage_diagnostics_cannot_replace_work_or_cleanup(stage_case, fault): + _, observe, events, state = stage_case(fault=fault) + result, records = await observe() + assert events.count("stage_error") == 1 # Causal baseline failure, not a new helper lookup. + assert result is state.first and len(state.commits) == 1 + assert len(state.explain) == (0 if fault in {"noncancel", "fatal"} else 1) + assert "restoration" in events and "public close" in events + assert ("cleanup_error" in events) is (fault == "explain") + assert any(record["kind"] == "freshplan_after_failure" for record in records) is ( + fault == "artifact" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("plan_fault", ["oversized", "deep", "nonfinite"]) +async def test_actual_stage_rejects_unsafe_plan_and_retains_cancellation(stage_case, plan_fault): + _, observe, events, state = stage_case(plan_fault=plan_fault) + result, records = await observe() + assert any(record["kind"] == "stage_error" for record in records) + assert result is state.first and len(state.explain) == 1 + assert not any(record["kind"] == "freshplan_after_failure" for record in records) + assert "triggers_restored" in events and "public close" in events + + +@pytest.mark.parametrize("fault", ["healthy", None]) +def test_actual_stage_reaches_main_verdict_and_retains_cleanup( + monkeypatch, tmp_path, capsys, stage_case, fault +): + from server.cli import _ch + + output, _ = _main_case(monkeypatch, tmp_path, None) + run, _, events, state = stage_case(fault=fault) + monkeypatch.setattr(_ch, "run_connected", lambda *_args, **_kwargs: asyncio.run(run())) + assert gate.main() == (0 if fault == "healthy" else 1) + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == ("PASS" if fault == "healthy" else "INVALID") + assert len(state.commits) == 1 and "triggers_restored" in events and "public close" in events + if fault == "healthy": + assert not state.explain and "settled" in events and "arm" in events + assert not any(kind in events for kind in ("stage_error", "freshplan_after_failure")) + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "driver_cleanup" + else: + assert "stage_error" in events and len(state.explain) == 1 + assert verdict["error_type"] == "PostgresUnavailableError" + assert verdict["failure"]["driver"]["outcome_unknown"] is False + assert json.loads(output.read_text().splitlines()[-1])["kind"] == "invalid" + assert SECRET not in json.dumps(verdict) + + +@pytest.mark.parametrize( + "message,reason,fault", + [(message, reason, None) for message, reason in MESSAGES] + + [(MESSAGES[0][0], "statement_timeout", fault) for fault in ("enum", "sqlstate", "extra")], +) +def test_native_action_reads_only_closed_cancellation_evidence( + keeper_config_case, message, reason, fault +): + native, instance, owner, _, _ = keeper_config_case + instance.s["python"] = "unused" + metadata = native_failure_metadata(canceled(message)) + cause = metadata["driver"]["causes"][0] + assert cause.get("cancel_reason") == reason # Baseline misses the actual server reason. + expected = deepcopy(metadata) + if fault == "enum": + cause["cancel_reason"] = SECRET + elif fault == "sqlstate": + cause["sqlstate"] = "08006" + elif fault == "extra": + cause["message"] = SECRET + native.write(instance.w / "pg_cell-created.json.failure.json", metadata) + original = native.Refused(SECRET) + + def refused(*_args, **_kwargs): + raise original + + owner.command = refused + with pytest.raises(native.Refused) as caught: + instance.action("create", role="pg_cell") + assert caught.value is original + assert original.native == (expected if fault is None else None) + assert SECRET not in json.dumps(original.native) From 184125564937483f7f056ad4f7f296657dbcfee0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 10:01:57 +0000 Subject: [PATCH 161/181] Exercise absent head locks at the native staging boundary Keep the production query and statement budget while reproducing the shared-identity workload in an isolated temporary table. Exact-row, ordering and identity controls distinguish a timeout from an incorrect match. Signed-off-by: zlareb1 --- tests/integration/test_pg_authority.py | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index fdeb0889c4..eee1707490 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -5750,3 +5750,112 @@ async def offered(after: Any = None) -> list[pg_memory.Clearable]: } assert a["value"] == "[REDACTED]" and a["redaction_reason"].startswith("leak") assert a["redacted_at"] and a["content_sha256"] != b["content_sha256"] + + +async def test_exact_head_locks_bound_a_large_absent_batch(pg_conn: Any, org: str) -> None: + """Absent keys sharing an identity do not exhaust the commit's statement budget.""" + from server.core.hashing import sha256_value + from server.memory.store import pg_memory + from server.memory.store.rows import ( + IDENTITY_COLUMNS, + MemoryRow, + build_first_version_row, + build_memory_version_row, + written_now, + ) + + namespace = "head-locks" + prefix = "gate-delivery-" + "1" * 32 + first_key = prefix + "-0" + template = pg_memory.encode_row( + build_memory_version_row( + MemoryRow.from_snapshot( + build_first_version_row( + { + "org": org, + "namespace": namespace, + "agent": "bench", + "memory_key": first_key, + "memory_type": "semantic", + "value": json.dumps(""), + "version": 1, + "content_sha256": sha256_value(""), + "actor": "process:head-lock-test", + "source_class": "asserted", + }, + now=written_now(), + ) + ) + ) + ) + assert pg_memory._LOCK_HEADS.count("loom.memory_current") == 1 + query = pg_memory._LOCK_HEADS.replace("loom.memory_current", "pg_temp.head_lock_candidates") + async with pg_conn.transaction(): + await pg_conn.execute( + "CREATE TEMP TABLE head_lock_candidates " + "(LIKE loom.memory_current INCLUDING ALL) WITH (fillfactor = 90) ON COMMIT DROP" + ) + # One statement builds full persisted rows without per-row network calls. + inserted = await pg_conn.execute( + "WITH keys AS (" + " SELECT i, $2::text || '-' || i::text AS key, " + "lpad(to_hex(i + 1), 32, '0')::uuid AS id FROM generate_series(0, 15699) AS i" + "), prefixes AS (" + " SELECT *, 'Cell trigger measurement ' || key || ': ' AS prefix FROM keys" + "), payloads AS (" + " SELECT *, prefix || left(repeat('abcdefghijklmnopqrstuvwxyz ', 4096), " + "4096 - length(prefix)) AS payload FROM prefixes" + "), stored AS (" + " SELECT *, to_json(payload)::text AS value, " + "encode(sha256(convert_to(to_json(payload)::text, 'UTF8')), 'hex') AS hash " + "FROM payloads) " + "INSERT INTO pg_temp.head_lock_candidates " + "(org, namespace, agent, user_id, session_id, memory_key, memory_id, " + "version, seq, content_sha256, row) " + "SELECT $1::jsonb->>'org', $1::jsonb->>'namespace', $1::jsonb->>'agent', " + "$1::jsonb->>'user_id', $1::jsonb->>'session_id', key, id, 1, i + 1, hash, " + "$1::jsonb || jsonb_build_object('memory_key', key, 'memory_id', id::text, " + "'value', value, 'content_sha256', hash) FROM stored", + template, + prefix, + ) + assert inserted == "INSERT 0 15700" + await pg_conn.execute("ANALYZE pg_temp.head_lock_candidates") + await pg_conn.execute("SET LOCAL statement_timeout = 5000") + await pg_conn.execute("SET LOCAL lock_timeout = 2000") + expected = {} + for index in (2, 10, 1): + key = f"{prefix}-{index}" + start = f"Cell trigger measurement {key}: " + payload = start + ("abcdefghijklmnopqrstuvwxyz " * 4096)[: 4096 - len(start)] + row = { + **template, + "memory_key": key, + "memory_id": str(uuid.UUID(int=index + 1)), + "value": json.dumps(payload), + "content_sha256": sha256_value(payload), + } + identity = pg_memory.identity_key(row) + expected[identity] = {**dict(zip(IDENTITY_COLUMNS, identity, strict=True)), "row": row} + matched = await pg_conn.fetch(query, *pg_memory._key_arrays(list(expected))) + assert [dict(row) for row in matched] == [expected[key] for key in sorted(expected)] + identity = next(iter(expected)) + changed = [ + tuple( + "different-" + value if position == axis else value + for position, value in enumerate(identity) + ) + for axis in range(6) + ] + assert await pg_conn.fetch(query, *pg_memory._key_arrays(sorted(changed))) == [] + absent_prefix = "gate-delivery-" + "2" * 32 + absent = sorted( + (org, namespace, "bench", "-", "-", f"{absent_prefix}-{index}") for index in range(2500) + ) + arguments = pg_memory._key_arrays(absent) + plan = await pg_conn.fetchval("EXPLAIN (FORMAT JSON) " + query, *arguments) + try: + assert await pg_conn.fetch(query, *arguments) == [] + except Exception as error: # The synthetic plan must not replace the native failure. + error.add_note("Synthetic head-lock plan before query: " + json.dumps(plan)) + raise From 310c407d819aa71afe9071e490904b029b6a1e16 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 10:21:23 +0000 Subject: [PATCH 162/181] Tracker feed holds native admission through owned cleanup Enroll before accepting org work, retain the permanent lifecycle through record cleanup, and preserve the first failure through both store closes. Classify the external feed table builder by its exact SQL sink. Signed-off-by: zlareb1 --- server/cli/tracker_feed.py | 106 ++++--- tests/unit/test_cell_control_census.py | 4 + tests/unit/test_tracker_feed_cli.py | 31 ++- tests/unit/test_tracker_feed_native_owner.py | 276 +++++++++++++++++++ 4 files changed, 371 insertions(+), 46 deletions(-) create mode 100644 tests/unit/test_tracker_feed_native_owner.py diff --git a/server/cli/tracker_feed.py b/server/cli/tracker_feed.py index 5b6e393154..759ea10c6d 100644 --- a/server/cli/tracker_feed.py +++ b/server/cli/tracker_feed.py @@ -7,6 +7,7 @@ import json import os import sys +from contextlib import suppress from typing import Any from server.core.github_values import FeedInputError @@ -78,8 +79,9 @@ async def _connection(args: argparse.Namespace, endpoint: Any, deployment: Any) try: await connect_external(client, deployment=deployment, orgs=(args.org,)) await _validate_writer(client, endpoint.database) - except BaseException: - await client.disconnect() + except BaseException: # Keep the opening failure through record cleanup. + with suppress(BaseException): + await client.disconnect() raise return client @@ -101,8 +103,9 @@ async def _validate_writer(client: Any, database: str) -> None: async def _run(args: argparse.Namespace) -> dict[str, Any]: from types import SimpleNamespace - from server.cli._ch import connect_checked, disconnect + from server.cli._ch import check_acceptance, connect_verified, disconnect from server.core.ch import get_client, set_query_tag + from server.insights.app.provider_command import model_command from server.settings import settings from server.tracker.app import github_feed as feed from server.tracker.store import github_feed as store @@ -113,48 +116,63 @@ async def _run(args: argparse.Namespace) -> dict[str, Any]: deployment = get_client() record = None try: - await connect_checked(deployment, orgs=(args.org,)) - owner = await store.database_uuid(deployment) - allowed: set[str] = set() - if args.feed_command == "init": - endpoint = SimpleNamespace( - host=args.host, - port=args.port, - secure=not args.insecure, - database=args.database, - ) - else: - endpoint, allowed, _reader = await _registered(args) - record = await _connection(args, endpoint, deployment) - if args.feed_command == "init": - await feed.initialize(record, args.org, owner) - return {"initialized": True} - if args.feed_command == "configure": - if not args.read_only_token: - raise FeedInputError("token permission attestation requires --read-only-token") - await feed.configure( - record, - org=args.org, - owner_uuid=owner, - repo=args.repo, - token=_environment(args.token_env), - allowed_repos=allowed, - max_pages=args.max_pages, - ) - return {"configured": True} - return await feed.sync( - record, - org=args.org, - owner_uuid=owner, - repo=args.repo, - allowed_repos=allowed, - ) - finally: - try: - if record is not None: - await record.disconnect() - finally: + await connect_verified(deployment) + async with model_command(): + try: + await check_acceptance(orgs=(args.org,)) + owner = await store.database_uuid(deployment) + allowed: set[str] = set() + if args.feed_command == "init": + endpoint = SimpleNamespace( + host=args.host, + port=args.port, + secure=not args.insecure, + database=args.database, + ) + else: + endpoint, allowed, _reader = await _registered(args) + record = await _connection(args, endpoint, deployment) + if args.feed_command == "init": + await feed.initialize(record, args.org, owner) + result: dict[str, Any] = {"initialized": True} + elif args.feed_command == "configure": + if not args.read_only_token: + raise FeedInputError( + "token permission attestation requires --read-only-token" + ) + await feed.configure( + record, + org=args.org, + owner_uuid=owner, + repo=args.repo, + token=_environment(args.token_env), + allowed_repos=allowed, + max_pages=args.max_pages, + ) + result = {"configured": True} + else: + result = await feed.sync( + record, + org=args.org, + owner_uuid=owner, + repo=args.repo, + allowed_repos=allowed, + ) + except BaseException: # Keep work failure through the owned record close. + if record is not None: + with suppress(BaseException): + await record.disconnect() + raise + else: + if record is not None: + await record.disconnect() + except BaseException: # Keep native work or retirement failure through store cleanup. + with suppress(BaseException): await disconnect(deployment) + raise + else: + await disconnect(deployment) + return result def _command(args: argparse.Namespace) -> int: diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index 64359d1293..1bbb66e2f0 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -154,6 +154,10 @@ def test_cell_control_migrations_are_classified(): # pg_migrate writes only its named migration ledger, and prepare's nonce writes only # the probe lock table. None is a cell acceptance or directory control write. GENERIC_SQL_EXCLUSIONS = { + # The allowlisted feed tables belong to the separate verified record database. + "server/tracker/store/github_feed.py": { + ("create_table", "CREATE TABLE IF NOT EXISTS", "{name}"): 1 + }, "server/cli/cell_prepare.py": {("_write_nonces", "INSERT INTO", "{leases.LOCK_TABLE}"): 1}, "server/code/store/index/prefix_twin.py": {("fill_sql", "INSERT INTO", "{table}"): 1}, "server/core/ch.py": {("_fenced_statement", "INSERT INTO", "{qualify(table)}"): 1}, diff --git a/tests/unit/test_tracker_feed_cli.py b/tests/unit/test_tracker_feed_cli.py index 251fa45116..b127db3ce4 100644 --- a/tests/unit/test_tracker_feed_cli.py +++ b/tests/unit/test_tracker_feed_cli.py @@ -9,6 +9,29 @@ from server.core.github_values import FeedInputError +@pytest.fixture +def native_lifetime(monkeypatch): + from server.core import retirement + from server.core.model_work_gate import ModelWorkGate + from server.insights.app import provider_command + + gate = retirement.get_gate() + model_gate = ModelWorkGate() + + async def enroll(): + gate.enrolled = True + + async def finish(): + gate.stopped() + + # These store-admission cases keep the real owner contexts with fake native I/O. + monkeypatch.setattr(provider_command, "register_process", enroll) + monkeypatch.setattr(provider_command, "load_active_configuration", AsyncMock()) + monkeypatch.setattr(provider_command, "_adopt_overrides", AsyncMock(return_value=True)) + monkeypatch.setattr(provider_command, "_finish_participant", finish) + monkeypatch.setattr(provider_command, "get_model_work_gate", lambda: model_gate) + + def test_missing_credential_names_only_the_environment_variable(monkeypatch): monkeypatch.delenv("FEED_TEST_MISSING_SECRET", raising=False) with pytest.raises(FeedInputError, match="FEED_TEST_MISSING_SECRET"): @@ -140,7 +163,9 @@ async def query_all(sql, params=None): @pytest.mark.asyncio @pytest.mark.parametrize("refusal", ["authority", "cell"]) -async def test_init_refuses_deployment_admission_before_record_or_write(monkeypatch, refusal): +async def test_init_refuses_deployment_admission_before_record_or_write( + monkeypatch, refusal, native_lifetime +): from server.cli import _ch from server.core import ch, pg, pg_profile from server.settings import settings @@ -189,7 +214,9 @@ async def test_init_refuses_deployment_admission_before_record_or_write(monkeypa @pytest.mark.asyncio -async def test_init_checks_actual_deployment_before_external_record_and_closes_both(monkeypatch): +async def test_init_checks_actual_deployment_before_external_record_and_closes_both( + monkeypatch, native_lifetime +): from server.cli import _ch from server.core import ch, hosts, pg, pg_profile from server.settings import settings diff --git a/tests/unit/test_tracker_feed_native_owner.py b/tests/unit/test_tracker_feed_native_owner.py new file mode 100644 index 0000000000..779da55d50 --- /dev/null +++ b/tests/unit/test_tracker_feed_native_owner.py @@ -0,0 +1,276 @@ +"""Tracker record work belongs to the deployment's entire native command lifetime.""" + +import asyncio +from types import SimpleNamespace + +import pytest + +from server.cli import _ch +from server.cli import tracker_feed as cli +from server.core import background +from server.core.model_work_gate import ModelChangeInProgress +from server.insights.app import provider_command as command +from tests.unit.test_cli_native_first_error import FIRST +from tests.unit.test_cli_native_first_error import native as native + + +@pytest.fixture +def tracker(native, monkeypatch): + from server.core import ch, hosts, pg_profile + from server.settings import settings + from server.tracker.app import github_feed as feed + from server.tracker.store import github_feed as store + + state = SimpleNamespace(native=native, strict=True, keepers=[]) + record = SimpleNamespace(connected=False) + endpoint = SimpleNamespace(host="record.example", database="record", port=8443, secure=True) + owner = "00000000-0000-0000-0000-000000000001" + arguments = dict( + org="team", + host=endpoint.host, + port=endpoint.port, + insecure=False, + database=endpoint.database, + writer_user="writer", + writer_password_env="TRACKER_OWNER_TEST_PASSWORD", + source="registered", + repo="team/project", + token_env="TRACKER_OWNER_TEST_TOKEN", + read_only_token=True, + max_pages=5, + ) + + def owned(): + if state.strict: + assert native.gate.role == "command" and native.gate.enrolled + assert native.gate.active == native.model_gate.active == 1 + assert not native.gate.closed and native.client.connected + + async def verify(client): + assert client is native.client + if "verify" not in native.events: + assert not native.gate.enrolled + native.event("verify") + + async def keeper(loaded): + state.keepers.append(asyncio.current_task()) + return await actual_keeper(loaded) + + async def deployment_owner(client): + assert client is native.client + owned() + native.event("owner") + return owner + + async def registered(args): + assert args.org == "team" and args.source == "registered" + owned() + native.event("registered") + return endpoint, {"team/project"}, "reader" + + async def vet(host, **kwargs): + assert host == endpoint.host + owned() + native.event("vet") + return "", "203.0.113.1" + + async def connect_record(): + owned() + native.event("record-connect") + record.connected = True + + async def close_record(): + owned() + await asyncio.sleep(0) + owned() + record.connected = False + native.event("record-close") + + async def validate(client, database): + assert client is record and database == endpoint.database + owned() + native.event("validate") + + async def work(): + owned() + native.event("work-enter") + await asyncio.sleep(0) + owned() + native.event("work") + + async def initialize(client, org, owner_uuid): + assert (client, org, owner_uuid) == (record, "team", owner) + await work() + + async def configure(client, **kwargs): + assert client is record + assert kwargs == dict( + org="team", + owner_uuid=owner, + repo="team/project", + token="token", + allowed_repos={"team/project"}, + max_pages=5, + ) + await work() + + async def sync(client, **kwargs): + assert client is record + assert kwargs == dict( + org="team", owner_uuid=owner, repo="team/project", allowed_repos={"team/project"} + ) + await work() + return {"synced": 1, "unchanged": 2} + + actual_keeper = command._keep_overrides + monkeypatch.setattr(command, "_keep_overrides", keeper) + monkeypatch.setattr(settings, "tracker_org", "team") + monkeypatch.setattr(pg_profile, "check_configured", verify) + monkeypatch.setattr(store, "database_uuid", deployment_owner) + monkeypatch.setattr(cli, "_registered", registered) + monkeypatch.setattr(hosts, "vet_host", vet) + monkeypatch.setattr(ch, "ExternalCHClient", lambda **kwargs: record) + monkeypatch.setattr(cli, "_validate_writer", validate) + monkeypatch.setattr(feed, "initialize", initialize) + monkeypatch.setattr(feed, "configure", configure) + monkeypatch.setattr(feed, "sync", sync) + monkeypatch.setenv("TRACKER_OWNER_TEST_PASSWORD", "password") + monkeypatch.setenv("TRACKER_OWNER_TEST_TOKEN", "token") + record.connect, record.disconnect = connect_record, close_record + state.record, state.endpoint = record, endpoint + state.args = lambda mode: SimpleNamespace(feed_command=mode, **arguments) + return state + + +@pytest.fixture +def cleanup_tracker(tracker, monkeypatch): + # Isolate cleanup defects from the separate missing-participant defect. + tracker.strict = False + + async def acceptance(orgs): + assert tuple(orgs) == ("team",) + tracker.native.event("acceptance") + + monkeypatch.setattr(_ch, "check_orgs", acceptance) + return tracker + + +def assert_closed(state): + assert not state.record.connected and not state.native.client.connected + assert state.native.gate.active == state.native.model_gate.active == 0 + assert not state.native.gate.producers + assert not background._registered_tasks + assert all(task.done() for task in state.keepers) + + +@pytest.mark.parametrize("mode", ["init", "configure", "sync"]) +def test_tracker_run_owns_acceptance_all_work_and_record_close_until_certified(tracker, mode): + result = asyncio.run(cli._run(tracker.args(mode))) + assert ( + result + == { + "init": {"initialized": True}, + "configure": {"configured": True}, + "sync": {"synced": 1, "unchanged": 2}, + }[mode] + ) + events = tracker.native.events + assert events.index("verify") < events.index("enroll") < events.index("configuration") + assert events.index("configuration") < events.index("acceptance") < events.index("owner") + assert events.index("owner") < events.index("record-connect") < events.index("work-enter") + assert events.index("work") < events.index("record-close") < events.index("stop") + assert events[-3:] == ["certificate", "pg", "ch"] + assert events.count("certificate") == 1 and tracker.native.gate.quiescent + assert tracker.keepers + assert_closed(tracker) + + +def test_tracker_run_refuses_closed_model_admission_before_record_or_work(tracker): + tracker.native.model_gate.close("transition") + with pytest.raises(ModelChangeInProgress): + asyncio.run(cli._run(tracker.args("sync"))) + assert "record-connect" not in tracker.native.events + assert "work-enter" not in tracker.native.events + assert_closed(tracker) + + +@pytest.mark.parametrize("refusal", ["verify", "acceptance"]) +def test_tracker_run_still_refuses_pairing_or_org_before_record_work(cleanup_tracker, refusal): + state = cleanup_tracker + first = ValueError("pairing") if refusal == "verify" else _ch.OrgNotHeld("org refused") + state.native.errors[refusal] = first + with pytest.raises(type(first)) as caught: + asyncio.run(cli._run(state.args("init"))) + assert caught.value is first + assert "owner" not in state.native.events and "record-connect" not in state.native.events + assert "work-enter" not in state.native.events + assert_closed(state) + + +@pytest.mark.parametrize("first_class", FIRST) +def test_tracker_run_preserves_work_failure_through_both_failed_closes( + cleanup_tracker, first_class +): + state = cleanup_tracker + first = first_class("feed work") + state.native.errors.update( + work=first, **{"record-close": OSError("record close"), "pg": OSError("deployment close")} + ) + with pytest.raises(first_class) as caught: + asyncio.run(cli._run(state.args("sync"))) + assert caught.value is first + events = state.native.events + assert events.index("work") < events.index("record-close") < events.index("pg") + assert events[-2:] == ["pg", "ch"] and events.count("record-close") == 1 + assert_closed(state) + + +def test_tracker_run_record_close_failure_stays_first_when_deployment_close_also_fails( + cleanup_tracker, +): + state = cleanup_tracker + first = ValueError("record close") + state.native.errors.update(**{"record-close": first, "pg": OSError("deployment close")}) + with pytest.raises(ValueError) as caught: + asyncio.run(cli._run(state.args("sync"))) + assert caught.value is first + assert state.native.events[-2:] == ["pg", "ch"] + assert state.native.events.count("record-close") == 1 + assert_closed(state) + + +@pytest.mark.parametrize("first_class", [ValueError, asyncio.CancelledError]) +def test_tracker_connection_preserves_validation_failure_when_record_close_fails( + cleanup_tracker, first_class +): + state = cleanup_tracker + first = first_class("writer validation") + state.native.errors.update(validate=first, **{"record-close": OSError("record close")}) + + async def run(): + try: + with pytest.raises(first_class) as caught: + await cli._connection(state.args("init"), state.endpoint, state.native.client) + assert caught.value is first + assert state.native.events[-2:] == ["validate", "record-close"] + finally: + await _ch.disconnect(state.native.client) + + asyncio.run(run()) + assert_closed(state) + + +def test_tracker_connection_success_keeps_record_open_for_its_caller(cleanup_tracker): + state = cleanup_tracker + + async def run(): + try: + record = await cli._connection(state.args("init"), state.endpoint, state.native.client) + assert record is state.record and record.connected + assert state.native.events[-2:] == ["record-connect", "validate"] + await record.disconnect() + finally: + await _ch.disconnect(state.native.client) + + asyncio.run(run()) + assert_closed(state) From 81bf248b156bb012ae637d8482e90cd37c1461c6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 10:53:48 +0000 Subject: [PATCH 163/181] Lock memory heads through ordered exact-key probes A large absent batch can merge on a partial identity and exhaust the existing statement budget. Sort complete identities before correlated row locks so the batch cannot flatten into that join. Retain all six tenant predicates, the transaction and lock order, and the unchanged native regression and deadlines. Signed-off-by: zlareb1 --- server/memory/store/pg_memory.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 89e2d67c07..c152690f83 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -489,11 +489,26 @@ def _key_arrays(keys: Sequence[IdentityKey]) -> list[list[str]]: ON d.org = c.org AND d.namespace = c.namespace AND d.seq = c.seq """ +# Lock each exact head only after its six-column key has been sorted. +# FOR UPDATE keeps the lateral probe separate from the batch's join planning. _LOCK_HEADS = f""" SELECT c.org, c.namespace, c.agent, c.user_id, c.session_id, c.memory_key, c.row -{_EXACT_KEYS} +FROM ( + SELECT * + FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::text[]) + AS k(org, namespace, agent, user_id, session_id, memory_key) + ORDER BY k.org COLLATE "C", k.namespace COLLATE "C", k.agent COLLATE "C", + k.user_id COLLATE "C", k.session_id COLLATE "C", k.memory_key COLLATE "C" +) k +CROSS JOIN LATERAL ( + SELECT c.org, c.namespace, c.agent, c.user_id, c.session_id, c.memory_key, c.row + FROM loom.memory_current c + WHERE c.org = k.org AND c.namespace = k.namespace AND c.agent = k.agent + AND c.user_id = k.user_id AND c.session_id = k.session_id + AND c.memory_key = k.memory_key + FOR UPDATE OF c +) c {_KEY_ORDER} -FOR UPDATE OF c """ _UPSERT_HEADS = """ From 5278b3d0aaa0ca81d6f61bb5a4d53d322c27fc85 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 11:00:40 +0000 Subject: [PATCH 164/181] Collect native fixture resources after setup failures Register each owned database and session as it is acquired. Attempt every cleanup when setup or teardown fails and preserve the original setup or body error. Keep all native assertions and deadlines unchanged. Signed-off-by: zlareb1 --- tests/integration/test_cell_control_fence.py | 61 ++++++++++--------- .../test_directory_control_cleanup.py | 27 +++++--- tests/integration/test_pg_owned_cleanup.py | 24 +++++--- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index 2e45540fb4..3dbf64a4ab 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import json import os @@ -28,34 +29,38 @@ async def control_stores(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Any]: dsn = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") admin = await asyncpg.connect(dsn) - name = f"loom_fence_{uuid.uuid4().hex[:12]}" - await admin.execute(f'CREATE DATABASE "{name}"') - own = urlunsplit(urlsplit(dsn)._replace(path=f"/{name}")) - monkeypatch.setattr(settings, "router_pg_dsn", own) - monkeypatch.setattr( - settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() - ) - monkeypatch.setattr(directory, "_pool", None) - conn = await asyncpg.connect(own) - try: - await directory.migrate() - await conn.execute( - "INSERT INTO loom_router.cells (cell, is_first, host, deployment_id, credential_sha256, state) " - "VALUES ('first', true, 'first.example.test', 'dep-first', $1, 'attaching')", - "a" * 64, - ) - transport = httpx.ASGITransport(app=app.build_app()) - async with httpx.AsyncClient( - transport=transport, - base_url="http://test", - headers={"Authorization": f"Bearer {TOKEN}"}, - ) as client: - yield client, conn, own - finally: - await conn.close() - await directory.close() - await admin.execute(f'DROP DATABASE "{name}" WITH (FORCE)') - await admin.close() + async with contextlib.AsyncExitStack() as cleanup: + try: + cleanup.push_async_callback(admin.close) + name = f"loom_fence_{uuid.uuid4().hex[:12]}" + await admin.execute(f'CREATE DATABASE "{name}"') + cleanup.push_async_callback(admin.execute, f'DROP DATABASE "{name}" WITH (FORCE)') + own = urlunsplit(urlsplit(dsn)._replace(path=f"/{name}")) + monkeypatch.setattr(settings, "router_pg_dsn", own) + monkeypatch.setattr( + settings, "router_operator_sha256", hashlib.sha256(TOKEN.encode()).hexdigest() + ) + monkeypatch.setattr(directory, "_pool", None) + cleanup.push_async_callback(directory.close) + conn = await asyncpg.connect(own) + cleanup.push_async_callback(conn.close) + await directory.migrate() + await conn.execute( + "INSERT INTO loom_router.cells (cell, is_first, host, deployment_id, credential_sha256, state) " + "VALUES ('first', true, 'first.example.test', 'dep-first', $1, 'attaching')", + "a" * 64, + ) + transport = httpx.ASGITransport(app=app.build_app()) + async with httpx.AsyncClient( + transport=transport, + base_url="http://test", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) as client: + yield client, conn, own + except BaseException: # Owned cleanup cannot replace the first failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise def expected(row: dict[str, Any], *, generation: bool = True) -> dict[str, Any]: diff --git a/tests/integration/test_directory_control_cleanup.py b/tests/integration/test_directory_control_cleanup.py index c07ce79ac8..928517ddf3 100644 --- a/tests/integration/test_directory_control_cleanup.py +++ b/tests/integration/test_directory_control_cleanup.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import os import time from collections.abc import AsyncIterator @@ -24,16 +25,22 @@ async def directory_database(monkeypatch) -> AsyncIterator[Any]: admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") name, dsn = await create_database(admin) - monkeypatch.setattr(settings, "router_pg_dsn", dsn) - monkeypatch.setattr(directory, "_pool", None) - observer = await asyncpg.connect(dsn) - await observer.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") - try: - yield observer - finally: - await directory.close() - await observer.close(timeout=1) - await drop_database(admin, name) + async with contextlib.AsyncExitStack() as cleanup: + try: + cleanup.push_async_callback(drop_database, admin, name) + monkeypatch.setattr(settings, "router_pg_dsn", dsn) + monkeypatch.setattr(directory, "_pool", None) + observer = await asyncpg.connect(dsn) + cleanup.push_async_callback(observer.close, timeout=1) + cleanup.push_async_callback(directory.close) + await observer.execute( + "CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)" + ) + yield observer + except BaseException: # Owned cleanup cannot replace the first failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise async def locked(conn): diff --git a/tests/integration/test_pg_owned_cleanup.py b/tests/integration/test_pg_owned_cleanup.py index 62a339774a..0cb7ca110a 100644 --- a/tests/integration/test_pg_owned_cleanup.py +++ b/tests/integration/test_pg_owned_cleanup.py @@ -25,15 +25,21 @@ async def database() -> AsyncIterator[tuple[Any, Any]]: admin = os.environ.get("LOOM_PG_DSN") or pytest.skip("LOOM_PG_DSN not set") name, dsn = await create_database(admin) - conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) - observer = await asyncpg.connect(dsn) - try: - await conn.execute("CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)") - yield conn, observer - finally: - await conn.close(timeout=1) - await observer.close(timeout=1) - await drop_database(admin, name) + async with contextlib.AsyncExitStack() as cleanup: + try: + cleanup.push_async_callback(drop_database, admin, name) + conn = await asyncpg.connect(dsn, server_settings={"statement_timeout": "10000"}) + cleanup.push_async_callback(conn.close, timeout=1) + observer = await asyncpg.connect(dsn) + cleanup.push_async_callback(observer.close, timeout=1) + await conn.execute( + "CREATE TABLE probe(id int PRIMARY KEY); INSERT INTO probe VALUES(1)" + ) + yield conn, observer + except BaseException: # Owned cleanup cannot replace the first failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise async def sleeping(observer: Any, pid: int) -> None: From 32d6790ecb4b95b26f166f80460f2448a6af4060 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 11:19:46 +0000 Subject: [PATCH 165/181] Keep attach tenancy writes on their owning backend A scan can outlive its ATTACH backend and recreate tenancy through the pool after fenced cleanup. Use the existing owner transaction for each unchanged batch and stop after an unknown commit. Keep synthetic request state scoped in its test, account for the tracker owner, and attempt every native fixture cleanup. Signed-off-by: zlareb1 --- server/cli/cell_attach.py | 4 +- server/cli/cell_attach_pg.py | 15 +- tests/integration/test_pg_online_index.py | 16 +- .../test_pg_online_index_durability.py | 23 +- tests/integration/test_router_directory_pg.py | 15 +- tests/unit/test_assistant_mirror_boundary.py | 34 +- tests/unit/test_cell_attach_tenancy_owner.py | 306 ++++++++++++++++++ tests/unit/test_cell_operator.py | 3 +- tests/unit/test_cell_retirement.py | 1 + 9 files changed, 374 insertions(+), 43 deletions(-) create mode 100644 tests/unit/test_cell_attach_tenancy_owner.py diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index ea20827a35..e624059924 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -512,7 +512,7 @@ async def step4(self, xmax: int, topology: Any, names: Any) -> None: listed = await cell_scan.list_tables(self.ch, self.lock_conn, topology, names) found = await self.scan(topology, listed) orgs = set(found.orgs) - self.report["tenancy_rows_written"] = await apg.write_tenancy(orgs) + self.report["tenancy_rows_written"] = await apg.write_tenancy(self.owner, orgs) placed = 0 for batch in batches(orgs, self.row["deployment_id"]): await self.confirm_lock() @@ -600,7 +600,7 @@ async def _owned(self, owner: cell_control.Owner) -> int: found = await self.scan(topology, listed) if found.too_long: raise Refused(_too_long(found)) - written = await apg.write_tenancy(found.orgs) + written = await apg.write_tenancy(self.owner, found.orgs) self.log(f"step 2: {len(found.orgs)} orgs found, {written} tenancy rows written") await self.triggers() await self.precommit(found) diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index b2b751d8af..89717d5699 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -15,6 +15,7 @@ import time from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator from dataclasses import dataclass +from functools import partial from typing import TYPE_CHECKING, Any from server.cli.cell_ch import Refusal, Refused @@ -853,15 +854,17 @@ async def _wait_transactions(conn: Any, xmax: int, wait: Wait) -> None: # -- the tenancy rows ------------------------------------------------------------------- -async def write_tenancy(orgs: Iterable[str]) -> int: - """The tenancy rows of `orgs`, on a marked session, as one statement per thousand; the - rows written.""" +async def write_tenancy(owner: Owner, orgs: Iterable[str]) -> int: + """Write each thousand-org batch on the exact ATTACH backend, returning rows added.""" batch = sorted(orgs) written = 0 for i in range(0, len(batch), 1000): - status = await pg.execute( - "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", - batch[i : i + 1000], + status = await owner.transaction( + partial( + owner.conn.execute, + "INSERT INTO loom.tenancy (org) SELECT unnest($1::text[]) ON CONFLICT DO NOTHING", + batch[i : i + 1000], + ) ) written += int(status.split()[-1]) return written diff --git a/tests/integration/test_pg_online_index.py b/tests/integration/test_pg_online_index.py index 38638ad877..8edbc2bdbc 100644 --- a/tests/integration/test_pg_online_index.py +++ b/tests/integration/test_pg_online_index.py @@ -39,12 +39,16 @@ async def connect() -> AsyncIterator[Any]: finally: await conn.close() - try: - async with connect() as conn: - yield conn, connect - finally: - await directory.close() - await drop_database(admin, name) + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(drop_database, admin, name) + cleanup.push_async_callback(directory.close) + try: + async with connect() as conn: + yield conn, connect + except BaseException: # Owned cleanup cannot replace the first failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise def _prefix(tmp_path: Path) -> Path: diff --git a/tests/integration/test_pg_online_index_durability.py b/tests/integration/test_pg_online_index_durability.py index 5acb835065..d575b8863e 100644 --- a/tests/integration/test_pg_online_index_durability.py +++ b/tests/integration/test_pg_online_index_durability.py @@ -9,6 +9,7 @@ import asyncio import contextlib import os +import sys from collections.abc import AsyncIterator from pathlib import Path from urllib.parse import urlsplit @@ -206,11 +207,23 @@ async def pause_before_ledger(*args, **kwargs): == "t" ) finally: - if paused: - await _docker("unpause", STANDBY) + first = sys.exception() + try: + if paused: + await _docker("unpause", STANDBY) + except BaseException as error: # noqa: BLE001 — attempt all cleanup; keep first failure. + first = first or error for task in tasks: if not task.done(): task.cancel() - async with asyncio.timeout(5): - await asyncio.gather(*tasks, return_exceptions=True) - observer.terminate() + try: + async with asyncio.timeout(5): + await asyncio.gather(*tasks, return_exceptions=True) + except BaseException as error: # noqa: BLE001 — observer cleanup survives join failure. + first = first or error + try: + observer.terminate() + except BaseException as error: # noqa: BLE001 — preserve the original drill failure. + first = first or error + if first is not None: + raise first diff --git a/tests/integration/test_router_directory_pg.py b/tests/integration/test_router_directory_pg.py index 41f8d343c6..5901f18837 100644 --- a/tests/integration/test_router_directory_pg.py +++ b/tests/integration/test_router_directory_pg.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import json import os @@ -41,11 +42,15 @@ async def dsn(monkeypatch: pytest.MonkeyPatch): name, value = await create_database(admin_dsn) monkeypatch.setattr(settings, "router_pg_dsn", value) monkeypatch.setattr(directory, "_pool", None) - try: - yield value - finally: - await directory.close() - await drop_database(admin_dsn, name) + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(drop_database, admin_dsn, name) + cleanup.push_async_callback(directory.close) + try: + yield value + except BaseException: # Owned cleanup cannot replace the first failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise @pytest.fixture diff --git a/tests/unit/test_assistant_mirror_boundary.py b/tests/unit/test_assistant_mirror_boundary.py index bd51b0fbf3..6c989968dd 100644 --- a/tests/unit/test_assistant_mirror_boundary.py +++ b/tests/unit/test_assistant_mirror_boundary.py @@ -165,23 +165,23 @@ def test_mcp_failures_keep_the_mark_after_protocol_unwrapping( 500: RuntimeError("private detail"), }[status] monkeypatch.setattr(mcp_outcome, "_emit_mcp_event", lambda *a, **kw: None) - request_work.begin() - request_work.mark_mirrored() - with private_work(): - result = mcp_outcome._handler_failed( - error, - 1, - "loom_assistant_ask", - SimpleNamespace(name="assistant.ask", idempotent=False), - {}, - None, - _caller(), - "request", - "session", - rl_ns="notes", - rl_org="example", - elapsed_ms=1, - ) + with request_work.bind(request_work.RequestWork()): + request_work.mark_mirrored() + with private_work(): + result = mcp_outcome._handler_failed( + error, + 1, + "loom_assistant_ask", + SimpleNamespace(name="assistant.ask", idempotent=False), + {}, + None, + _caller(), + "request", + "session", + rl_ns="notes", + rl_org="example", + elapsed_ms=1, + ) assert next(iter(result)) == "mirrored" and result["mirrored"] is True if status == 403: body = result["result"] diff --git a/tests/unit/test_cell_attach_tenancy_owner.py b/tests/unit/test_cell_attach_tenancy_owner.py new file mode 100644 index 0000000000..019de4749f --- /dev/null +++ b/tests/unit/test_cell_attach_tenancy_owner.py @@ -0,0 +1,306 @@ +"""A scan cannot write tenancy after its exact ATTACH backend has ended.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock +from uuid import UUID + +import pytest + +from server.cli import cell_attach, cell_control, cell_scan +from server.cli import cell_attach_pg as apg +from server.core import cell +from server.settings import settings +from tests.unit.cell_control_owner import owner +from tests.unit.test_cell_attach import CURRENT, NEXT, FakeConn, _args +from tests.unit.test_cell_control import snapshot + + +class Store: + def __init__(self, phase: str, interruption: str): + self.mode = "recording" if phase == "recording" else "enforced" + self.first = {"is_first": True, "cell": "first"} + self.own = {"current_fp": CURRENT, "next_fp": NEXT} + self.tenancy = {"org-0000"} + self.guard = True + self.holder = "attach-1" + self.row = {**snapshot(), **self.own, "state": "attaching", "placed_orgs": 0} + self.phase, self.interruption = phase, interruption + self.paused, self.resume = asyncio.Event(), asyncio.Event() + self.batches: list[tuple[str, list[str]]] = [] + self.closed: list[str] = [] + self.claims = 0 + self.commits = 0 + + async def call(self, path: str, body: Any = None): + if not path: + return dict(self.row) + assert ( + body["expected"] + == cell_control.preconditions(self.row, claim=path == "/claim")["expected"] + ) + if path == "/claim": + self.claims += 1 + self.row["mutation_generation"] = str(UUID(int=self.claims + 1)) + elif path == "/limit": + self.row["closed"] = body["closed"] + elif path == "/attach/place": + self.row["placed_orgs"] += len(body["orgs"]) + return {**self.row, "placed": len(body["orgs"])} + else: + assert path == "/attach/enforced" + return dict(self.row) + + async def pause_after_batch(self): + if self.interruption == "between" and len(self.batches) == 1: + self.paused.set() + await self.resume.wait() + + +class Session(FakeConn): + def __init__(self, store: Store, identity: str): + super().__init__() + self.store, self.identity = store, identity + self.dead = False + self.pending: set[str] | None = None + self.has_insert = False + + def alive(self): + if self.dead: + raise ConnectionError("exact ATTACH backend ended") + if self.identity != "pool" and self.store.holder != self.identity: + raise ConnectionError("exact ATTACH backend no longer owns its session") + + def _answer(self, sql: str, args: tuple[Any, ...]): + self.alive() + store = self.store + if sql == apg._HELD: + return store.holder == self.identity + if sql == cell._STATE: + return { + "is_first": store.first["is_first"] if store.first else None, + "cell": store.first["cell"] if store.first else None, + "mode": store.mode, + "has_own": store.own is not None, + "has_tenancy": bool(store.tenancy), + } + if sql == "SELECT deployment_id FROM loom.authority": + return "dep" + if sql == "SELECT mode FROM loom.tenancy_mode": + return store.mode + if sql == "SELECT is_first, cell FROM loom.cell_first": + return store.first + if sql == "SELECT current_fp, next_fp FROM loom.cell_self": + return store.own + if sql == "SELECT EXISTS(SELECT 1 FROM loom.tenancy)": + return bool(store.tenancy) + if "FROM pg_trigger" in sql: + return [{"tgname": "tenancy_record", "relname": "memory_current"}] + return super()._answer(sql, args) + + async def execute(self, sql: str, *args: Any): + self.alive() + store = self.store + if sql == "BEGIN": + self.pending = set(store.tenancy) + elif sql.startswith("INSERT INTO loom.tenancy"): + batch = list(args[0]) + target = store.tenancy if self.pending is None else self.pending + added = len(set(batch) - target) + target.update(batch) + store.batches.append((self.identity, batch)) + self.has_insert = True + if self.pending is None: + await store.pause_after_batch() + return f"INSERT 0 {added}" + elif sql == "COMMIT": + if self.pending is not None: + store.tenancy = self.pending + self.pending = None + if self.has_insert: + self.has_insert = False + store.commits += 1 + if store.interruption == "unknown": + raise OSError("tenancy COMMIT reply lost") + await store.pause_after_batch() + elif sql == "ROLLBACK": + self.pending = None + elif sql.startswith("DELETE FROM loom."): + table = sql.removeprefix("DELETE FROM loom.") + if table == "tenancy": + store.tenancy.clear() + if self.pending is not None: + self.pending.clear() + else: + assert table in {"cell_first", "tenancy_mode", "cell_self"} + field = {"cell_first": "first", "tenancy_mode": "mode", "cell_self": "own"}[table] + setattr(store, field, None) + elif sql.startswith("DROP TRIGGER"): + store.guard = False + return await super().execute(sql, *args) + + def lose(self): + self.dead = True + self.pending = None + if self.store.holder == self.identity: + self.store.holder = "" + + async def close(self, **kwargs: Any): + self.store.closed.append(self.identity) + self.lose() + await super().close(**kwargs) + + def is_closed(self): + return self.dead + + +def prepare(monkeypatch, phase: str, interruption: str, count: int): + store = Store(phase, interruption) + raw, pooled = Session(store, "attach-1"), Session(store, "pool") + orgs = [f"org-{i:04}" for i in range(count)] + found = cell_scan.ScanResult( + orgs={org: {"postgres loom.memory_current"} for org in reversed(orgs)} + ) + logs: list[str] = [] + run = cell_attach.Run(AsyncMock(), _args(), SimpleNamespace(call=store.call), logs.append) + run.own_fp = (CURRENT, NEXT) + run.read_own_row = AsyncMock() + run.refusals = AsyncMock(return_value=(None, None, None)) + run.begin = AsyncMock() + run.precommit = AsyncMock() + run.durable = AsyncMock(return_value=100) + run.check_beats = AsyncMock() + run.cheap_checks = AsyncMock() + run.nonheartbeat_checks = AsyncMock() + run.final_checks = AsyncMock() + run.serving = AsyncMock(return_value={"state": "serving", "placed_orgs": count}) + + async def opened(owned, what): + assert what == "attach" + owned.conn = raw + await cell_control.check_lock(raw) + + async def scan(*args): + if interruption == "scan": + store.paused.set() + await store.resume.wait() + return found + + async def enforced(): + store.mode, run.mode = "enforced", "enforced" + + async def triggers(): + await cell_control.check_lock(raw) + + async def pooled_execute(sql, *args): + return await pooled.execute(sql, *args) + + @asynccontextmanager + async def dedicated(**kwargs): + yield raw + + async def watch(_self): + await asyncio.Future() + + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "document_s3_bucket", "") + monkeypatch.setattr(cell_control.Owner, "_open", opened) + monkeypatch.setattr(apg.pg, "execute", pooled_execute) + monkeypatch.setattr(apg.pg, "dedicated", dedicated) + monkeypatch.setattr(apg, "wait_transactions", AsyncMock()) + monkeypatch.setattr(cell_scan, "list_tables", AsyncMock(return_value=None)) + monkeypatch.setattr(cell_attach.beats.Watch, "run", watch) + run.scan, run.triggers, run.commit_mode = scan, triggers, enforced + if phase == "recording": + run.step4 = AsyncMock() + return store, raw, run, orgs, logs + + +async def lose_and_clean(store: Store, raw: Session): + raw.lose() + if store.phase == "recording": + store.holder = "cleanup-2" + cleanup = Session(store, "cleanup-2") + cleanup_owner = owner(cleanup) + try: + await cell_control.backout(cleanup_owner, store.call, dict(store.row)) + finally: + await cleanup_owner.close() + assert store.tenancy == set() + assert (store.mode, store.first, store.own, store.guard) == (None, None, None, False) + assert store.row["closed"] is True and store.row["placed_orgs"] == 0 + # First-cell enforced backout is forbidden: step 4 tests only backend loss. + return set(store.tenancy) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["recording", "step4"]) +@pytest.mark.parametrize("interruption", ["scan", "between"]) +async def test_lost_attach_cannot_write_tenancy_after_its_scan_or_first_batch( + monkeypatch, phase, interruption +): + store, raw, run, orgs, _ = prepare(monkeypatch, phase, interruption, 2001) + task = asyncio.create_task(run.run()) + paused = asyncio.create_task(store.paused.wait()) + try: + await asyncio.wait({task, paused}, return_when=asyncio.FIRST_COMPLETED) + if task.done(): + await task + assert store.paused.is_set() + preserved = await lose_and_clean(store, raw) + batches_at_loss = len(store.batches) + store.resume.set() + with pytest.raises((cell_control.Refused, ConnectionError)): + await task + assert store.tenancy == preserved + assert len(store.batches) == batches_at_loss + assert (set(orgs[batches_at_loss * 1000 :]) - preserved).isdisjoint(store.tenancy) + if phase == "recording": + assert (store.mode, store.first, store.own) == (None, None, None) + assert not store.tenancy and not store.guard + assert await cell.read_state(Session(store, "pool")) == "open" + finally: + store.resume.set() + if not task.done(): + task.cancel() + if not paused.done(): + paused.cancel() + await asyncio.gather(task, paused, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["recording", "step4"]) +async def test_healthy_attach_keeps_sorted_thousand_org_batches_and_insert_count( + monkeypatch, phase +): + store, _, run, orgs, logs = prepare(monkeypatch, phase, "healthy", 2001) + assert await run.run() == 0 + assert store.tenancy == set(orgs) + assert [org for _, batch in store.batches for org in batch] == orgs + assert [len(batch) for _, batch in store.batches] == [1000, 1000, 1] + assert store.claims == 1 and store.closed.count("attach-1") == 1 + if phase == "recording": + assert "step 2: 2001 orgs found, 2000 tenancy rows written" in logs + else: + assert run.report["tenancy_rows_written"] == 2000 + assert run.report["orgs_found"] == 2001 + assert run.report["placed_now"] == 2001 + + +@pytest.mark.asyncio +async def test_unknown_tenancy_commit_stops_before_a_later_batch_without_replay(monkeypatch): + store, _, run, orgs, _ = prepare(monkeypatch, "recording", "unknown", 2001) + error = None + try: + await run.run() + except OSError as caught: + error = caught + assert store.tenancy == set(orgs[:1000]) + assert len(store.batches) == 1 + assert str(error) == "tenancy COMMIT reply lost" + assert store.closed.count("attach-1") == 1 + assert store.claims == 1 diff --git a/tests/unit/test_cell_operator.py b/tests/unit/test_cell_operator.py index 5c37b78cb0..83a2ea143f 100644 --- a/tests/unit/test_cell_operator.py +++ b/tests/unit/test_cell_operator.py @@ -357,8 +357,7 @@ def test_a_cell_name_no_cell_can_have_is_refused_before_the_credential_is_sent( @pytest.mark.parametrize("command", ["which", "limit"]) def test_a_page_that_is_not_the_routers_answer_fails(monkeypatch, capsys, command) -> None: - """A URL that serves something else (an HTML page, with 200) is not read as the - router's answer; for a change, it may have been made.""" + """A malformed initial router answer refuses before sending a mutation.""" page = httpx.Response( 200, text="not the router", request=httpx.Request("GET", "https://x/") ) diff --git a/tests/unit/test_cell_retirement.py b/tests/unit/test_cell_retirement.py index 48517fb11e..f6b63bec70 100644 --- a/tests/unit/test_cell_retirement.py +++ b/tests/unit/test_cell_retirement.py @@ -429,6 +429,7 @@ def test_retirement_writers_and_accepting_command_roots_are_an_explicit_census() "cli/code_run.py", "cli/import_clickgap.py", "cli/import_recall.py", + "cli/tracker_feed.py", } assert plain == { "cli/sql_access.py", From 00fafb2da4aa8b3d74938937ea4a6176746ccfa6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 11:32:16 +0000 Subject: [PATCH 166/181] Document unproved attach lock cleanup A stalled backend can outlive bounded client cleanup. Require exact-holder verification when release cannot be proved. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 45b319c1d8..2edc42c9b9 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1476,7 +1476,7 @@ participate in this protocol. | A cell in `recording`, the directory unreadable | a new org answers 503; the orgs it holds keep serving | | A process stuck during first attach | before step 3, attach refuses after `--wait` and the cell stays `recording`; after it, attach refuses, leaving the cell `enforced` and `attaching`, and a rerun resumes | | A recording insert held open during first attach | the mode commit's tries wait behind its `FOR SHARE` lock within `--wait`; past it attach refuses and the cell stays `recording`; a rerun resumes | -| Attach stalls while it holds the mode table's lock | Postgres ends its session within seconds and releases the lock; new orgs answer within their deadline | +| Attach stalls while it holds the mode table's lock | the client attempts bounded cleanup; if backend or lock release is unproved, stop and verify the exact holder before retrying | | A transaction open since before step 4's snapshot | attach refuses once `--wait` runs out (a prepared one at once), naming what it waited on, and leaves `enforced` | | A CLI command with `LOOM_CELL` unset, started before attach's step 3 | it read the state while the first cell was `recording` and keeps writing ClickHouse after `serving`; the runbook sets `LOOM_CELL` on every CLI before attach, and `loom doctor --stray-orgs` names the orgs it left without a tenancy row | | A CLI of a build before the state read | it is not refused on an `enforced` cell; every CLI and CI image runs a build with the read before attach ([upgrades](../operations/upgrades.md#upgrade-procedure)) | From b0c7dd64d0377f279ff75392d8458b9cb493ae88 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 11:40:29 +0000 Subject: [PATCH 167/181] Align attach cleanup and pairing descriptions Backend termination can remain unproved after cleanup, and later-cell removal verifies paired stores. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 5 +++-- tests/unit/test_router_attach.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 2edc42c9b9..8072d5b3dc 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -731,8 +731,9 @@ it only while the mode is `recording`, reading the mode row `FOR SHARE`; it runs `READ COMMITTED` in its own transaction and commits before the request goes on. Attach's write of `enforced` and these inserts never cross: an insert already running when attach writes the mode is placed by step 4, one that starts after reads `enforced`, a stream of -inserts cannot hold the write off, a new-org request waits on attach for a few seconds -at most, and an attach that stalls holding the write's lock releases it within seconds. +inserts cannot hold the write off. If attach stalls while holding the mode lock, it +attempts bounded cleanup. A client abort does not prove that the lock was released; +stop and verify the exact holder before retrying. Attach takes `LOCK TABLE loom.tenancy_mode IN EXCLUSIVE MODE` (which waits for running inserts and queues later ones behind it) in short tries, in one transaction with server-side timeouts, then writes the mode and commits on its exact ATTACH backend. diff --git a/tests/unit/test_router_attach.py b/tests/unit/test_router_attach.py index 821272bcdf..8265ef9620 100644 --- a/tests/unit/test_router_attach.py +++ b/tests/unit/test_router_attach.py @@ -1203,7 +1203,7 @@ def test_remove_tells_an_older_router_from_an_unknown_cell_on_the_delete( cli_router: _Router, capsys: Any ) -> None: """A replica of an earlier build behind the same URL answers the DELETE alone.""" - cli_router.row = {**cli_router.row, "is_first": False} # no Postgres read + cli_router.row = {**cli_router.row, "is_first": False} cli_router.deleted = (404, {"error": "the router answers no such path"}) assert cell_cli._cmd_remove(_remove_args()) == 1 assert "not upgraded" in capsys.readouterr().err From 75892468fe1cec9f1544e05aa995d690d1137828 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 12:15:18 +0000 Subject: [PATCH 168/181] SCIM fixtures attempt every owned cleanup A failed global-row or Keeper cleanup must still allow remaining exact-resource cleanup and disconnect. Propagate the first teardown failure after all attempts. Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 24 ++++++++++----- .../integration/test_scim_sessions_live_ch.py | 29 +++++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 0ce6b29d70..7ba359657a 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -139,13 +139,23 @@ async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) try: yield made finally: - if made.emails: - await client.execute( - "ALTER TABLE loom.scim_users DELETE WHERE id IN {ids:Array(String)} " - "SETTINGS mutations_sync = 2", - params={"ids": [scim.user_id(e) for e in made.emails]}, - ) - await client.disconnect() + first_error: BaseException | None = None + try: + if made.emails: + await client.execute( + "ALTER TABLE loom.scim_users DELETE WHERE id IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + params={"ids": [scim.user_id(e) for e in made.emails]}, + ) + except BaseException as exc: # noqa: BLE001 — disconnect must still run + first_error = exc + try: + await client.disconnect() + except BaseException as exc: # noqa: BLE001 — preserve the first cleanup failure + if first_error is None: + first_error = exc + if first_error is not None: + raise first_error async def test_verified_reverse_owner_still_requires_its_own_email(people: People) -> None: diff --git a/tests/integration/test_scim_sessions_live_ch.py b/tests/integration/test_scim_sessions_live_ch.py index 524d7d6ffd..9f78604c9b 100644 --- a/tests/integration/test_scim_sessions_live_ch.py +++ b/tests/integration/test_scim_sessions_live_ch.py @@ -79,16 +79,27 @@ async def sessions(people: People, monkeypatch: pytest.MonkeyPatch) -> AsyncIter try: yield store finally: + first_error: BaseException | None = None for key in store.keys: - current = await claims.get(people.client, key) - if current is not None: - await claims._drop(people.client, key, current) - if store.families: - await people.client.execute( - "ALTER TABLE loom.revoked_tokens DELETE WHERE jti IN {ids:Array(String)} " - "SETTINGS mutations_sync = 2", - params={"ids": list(store.families)}, - ) + try: + current = await claims.get(people.client, key) + if current is not None: + await claims._drop(people.client, key, current) + except BaseException as exc: # noqa: BLE001 — attempt every owned key + if first_error is None: + first_error = exc + try: + if store.families: + await people.client.execute( + "ALTER TABLE loom.revoked_tokens DELETE WHERE jti IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + params={"ids": list(store.families)}, + ) + except BaseException as exc: # noqa: BLE001 — preserve the first cleanup failure + if first_error is None: + first_error = exc + if first_error is not None: + raise first_error async def test_real_refresh_round_trip_putback_and_rotation_preserve_email( From 5c6913eb1de9a43dfb1f482c085e358abeac2067 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 12:37:55 +0000 Subject: [PATCH 169/181] Cell configuration bounds include operator placements Reserve the largest live or operator request with the same body limit. Keep parser diagnostics observable and attempt independent fixture cleanup after setup or teardown failures. Signed-off-by: zlareb1 --- server/router/fingerprints.py | 23 +- .../integration/test_cell_place_durability.py | 48 +++- .../test_cell_update_participants_ch.py | 203 +++++++------- tests/integration/test_cell_update_run_pg.py | 259 +++++++++--------- .../test_cell_fingerprint_reservations.py | 66 ++++- .../unit/test_code_repo_entry_clone_fields.py | 9 +- 6 files changed, 364 insertions(+), 244 deletions(-) diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py index 884948b609..25fea1e3f8 100644 --- a/server/router/fingerprints.py +++ b/server/router/fingerprints.py @@ -168,12 +168,27 @@ def same(a: dict[str, Any], b: dict[str, Any]) -> bool: def placement_refusal(cell_name: str, name: str, value: dict[str, Any]) -> str: - """Why a fingerprint is too large for the cell's live placements ('' when it is not): a - placement body (`{org, cell, fingerprint}`, the org at its longest) past the router's - MAX_BODY_BYTES would refuse every new org; the largest fields are named.""" + """Why a fingerprint cannot fit both live and operator placement bodies. + + Reserve the longest escaped org in each envelope and name only the largest fields. + """ # The default encoding is the longest: spaces after separators, non-ASCII escaped. size = ( - len(json.dumps({"org": "", "cell": cell_name, "fingerprint": value})) + _PLACEMENT_ORG_MAX + max( + len(json.dumps({"org": "", "cell": cell_name, "fingerprint": value})), + len( + json.dumps( + { + "org": "", + "deployment_id": value["fields"].get( + "deployment_id", cell.DEFAULTS["deployment_id"] + ), + "fingerprint": value, + } + ) + ), + ) + + _PLACEMENT_ORG_MAX ) if size <= MAX_BODY_BYTES: return "" diff --git a/tests/integration/test_cell_place_durability.py b/tests/integration/test_cell_place_durability.py index 1d788f47d6..5d8c369c01 100644 --- a/tests/integration/test_cell_place_durability.py +++ b/tests/integration/test_cell_place_durability.py @@ -112,6 +112,36 @@ async def seed(self, dsn: str, sql: str, *, marked: bool = False) -> None: conn.terminate() +async def _cleanup_drill(state: Drill | None, observer: Any) -> None: + errors: list[BaseException] = [] + if state is not None: + try: + if state.paused: + await state.resume() + except BaseException as exc: # noqa: BLE001 — remaining owned cleanup must run + errors.append(exc) + for task in state.tasks: + if not task.done(): + task.cancel() + deadline = asyncio.get_running_loop().time() + 15 + for close in ( + lambda: asyncio.gather(*state.tasks, return_exceptions=True), + directory.close, + pg.close_pool, + ): + try: + async with asyncio.timeout_at(deadline): + await close() + except BaseException as exc: # noqa: BLE001 — attempt each close within the shared bound + errors.append(exc) + try: + observer.terminate() + except BaseException as exc: # noqa: BLE001 — preserve the first cleanup failure + errors.append(exc) + if errors: + raise errors[0] + + @pytest.fixture async def drill(monkeypatch): if not os.environ.get("LOOM_CELL_PLACE_DRILL"): @@ -161,18 +191,12 @@ async def drill(monkeypatch): finally: pg.expect_deployment(previous_expected) cell._code_orgs = previous_orgs - finally: - if state is not None: - if state.paused: - await state.resume() - for task in state.tasks: - if not task.done(): - task.cancel() - async with asyncio.timeout(15): - await asyncio.gather(*state.tasks, return_exceptions=True) - await directory.close() - await pg.close_pool() - observer.terminate() + except BaseException: # Cleanup must not replace the setup failure. + with contextlib.suppress(BaseException): + await _cleanup_drill(state, observer) + raise + else: + await _cleanup_drill(state, observer) async def _initialize(drill: Drill) -> None: diff --git a/tests/integration/test_cell_update_participants_ch.py b/tests/integration/test_cell_update_participants_ch.py index 5fc5176c72..48e81c9495 100644 --- a/tests/integration/test_cell_update_participants_ch.py +++ b/tests/integration/test_cell_update_participants_ch.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import json import os from types import SimpleNamespace @@ -28,115 +29,125 @@ async def test_postcutoff_census_waits_for_certified_old_row_and_serving_success database = "update_" + uuid4().hex client = CHClient() await client.connect() - task = None with query_tag("test.cell_update.participants"): - await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) - try: - await client.execute( - "CREATE TABLE {db:Identifier}._provider_participants AS loom._provider_participants", - {"db": database}, - ) - for table in ("instances", "cell_heartbeats"): + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(client.disconnect) + try: + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + cleanup.push_async_callback( + client.execute, "DROP DATABASE {db:Identifier} SYNC", {"db": database} + ) await client.execute( - f"CREATE TABLE {{db:Identifier}}.{table} AS loom.{table} ENGINE=MergeTree ORDER BY instance_id", + "CREATE TABLE {db:Identifier}._provider_participants AS loom._provider_participants", {"db": database}, ) - monkeypatch.setattr(settings, "ch_database", database) - monkeypatch.setattr(settings, "ch_cluster", "") - old, new = uuid4(), uuid4() - proof = provider_process.certified(provider_process.host_proof()) - await client.insert( - "loom._provider_participants", - [{"participant_id": old, "state": "running", "version": 1, "host_proof": proof}], - ) - await client.execute( - "ALTER TABLE loom._provider_participants UPDATE heartbeat_at=now64(3)-toIntervalHour(25) WHERE 1 SETTINGS mutations_sync=2" - ) - initial = await update.census(client) - assert set(update.capture(list(initial.values()))) == {str(old)} - target = cell.as_value({"deployment_id": "target"}) - captured = asyncio.Event() - actual_census = update.census - reads = 0 - - async def census(ch): - nonlocal reads - result = await actual_census(ch) - reads += 1 - if reads == 1: - captured.set() - return result - - async def successor(): - await captured.wait() + for table in ("instances", "cell_heartbeats"): + await client.execute( + f"CREATE TABLE {{db:Identifier}}.{table} AS loom.{table} ENGINE=MergeTree ORDER BY instance_id", + {"db": database}, + ) + monkeypatch.setattr(settings, "ch_database", database) + monkeypatch.setattr(settings, "ch_cluster", "") + old, new = uuid4(), uuid4() + proof = provider_process.certified(provider_process.host_proof()) await client.insert( "loom._provider_participants", [ { "participant_id": old, - "state": "retired", - "version": 2, - "host_proof": proof, - }, - { - "participant_id": new, "state": "running", "version": 1, "host_proof": proof, - }, - ], - ) - instance = instances.row() | { - "instance_id": "successor", - "serving": 1, - "stopped": 0, - } - await client.insert("loom.instances", [instance]) - await client.insert( - "loom.cell_heartbeats", - [ - { - "instance_id": "successor", - "cell": "first", - "tenancy_mode": "enforced", - "fingerprint": json.dumps( - target - | { - "participant_id": str(new), - "serves": True, - "marked": True, - "router": True, - } - ), } ], ) + await client.execute( + "ALTER TABLE loom._provider_participants UPDATE heartbeat_at=now64(3)-toIntervalHour(25) WHERE 1 SETTINGS mutations_sync=2" + ) + initial = await update.census(client) + assert set(update.capture(list(initial.values()))) == {str(old)} + target = cell.as_value({"deployment_id": "target"}) + captured = asyncio.Event() + actual_census = update.census + reads = 0 + + async def census(ch): + nonlocal reads + result = await actual_census(ch) + reads += 1 + if reads == 1: + captured.set() + return result + + async def successor(): + await captured.wait() + await client.insert( + "loom._provider_participants", + [ + { + "participant_id": old, + "state": "retired", + "version": 2, + "host_proof": proof, + }, + { + "participant_id": new, + "state": "running", + "version": 1, + "host_proof": proof, + }, + ], + ) + instance = instances.row() | { + "instance_id": "successor", + "serving": 1, + "stopped": 0, + } + await client.insert("loom.instances", [instance]) + await client.insert( + "loom.cell_heartbeats", + [ + { + "instance_id": "successor", + "cell": "first", + "tenancy_mode": "enforced", + "fingerprint": json.dumps( + target + | { + "participant_id": str(new), + "serves": True, + "marked": True, + "router": True, + } + ), + } + ], + ) - monkeypatch.setattr(update, "census", census) - monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) - run = SimpleNamespace( - ch=client, - args=SimpleNamespace(cell="first", wait=5), - row={"is_first": True}, - owner=owner(FakeConn()), - ) - task = asyncio.create_task(successor()) - await update.settle(run, target) - await task - assert reads >= 2 - assert len(await actual_census(client)) == 2 - # A workers-only heartbeat cannot replace the serving target. - await client.execute("TRUNCATE TABLE loom.instances") - await client.insert( - "loom.instances", [instances.row() | {"instance_id": "successor", "serving": 0}] - ) - assert not await update.replacement( - client, run, await actual_census(client), {str(old)}, target - ) - finally: - if task is not None: - if not task.done(): - task.cancel() - await asyncio.gather(task, return_exceptions=True) - await client.execute("DROP DATABASE {db:Identifier} SYNC", {"db": database}) - await client.disconnect() + monkeypatch.setattr(update, "census", census) + monkeypatch.setattr(cell_control, "check_lock", AsyncMock()) + run = SimpleNamespace( + ch=client, + args=SimpleNamespace(cell="first", wait=5), + row={"is_first": True}, + owner=owner(FakeConn()), + ) + task = asyncio.create_task(successor()) + cleanup.push_async_callback(asyncio.gather, task, return_exceptions=True) + cleanup.callback(task.cancel) + await update.settle(run, target) + await task + assert reads >= 2 + assert len(await actual_census(client)) == 2 + # A workers-only heartbeat cannot replace the serving target. + await client.execute("TRUNCATE TABLE loom.instances") + await client.insert( + "loom.instances", [instances.row() | {"instance_id": "successor", "serving": 0}] + ) + assert not await update.replacement( + client, run, await actual_census(client), {str(old)}, target + ) + except BaseException: # Owned cleanup must not replace setup or body failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise diff --git a/tests/integration/test_cell_update_run_pg.py b/tests/integration/test_cell_update_run_pg.py index db1adc3c33..4ef957a07c 100644 --- a/tests/integration/test_cell_update_run_pg.py +++ b/tests/integration/test_cell_update_run_pg.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import json from argparse import Namespace @@ -157,36 +158,35 @@ async def test_cancelled_acceptance_releases_backend_and_rolls_back_tenancy(upda make, local_conn, directory_conn, owned, _, local_dsn = update_stores backend_pid = local_conn.get_server_pid() before = await cell_update_pg.read(local_conn, "first") - blocker = await asyncpg.connect(local_dsn) - transaction = blocker.transaction() - await transaction.start() - task = None - observer = None - try: - observer = await asyncpg.connect(local_dsn) - await blocker.fetchrow("SELECT id FROM loom.cell_self FOR UPDATE") - task = asyncio.create_task(make().start(owned)) - async with asyncio.timeout(4): - while not await observer.fetchval( - "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " - "AND wait_event_type='Lock' AND query LIKE 'UPDATE loom.cell_self%')", - backend_pid, - ): - assert not task.done() - await asyncio.sleep(0.005) - task.cancel("acceptance-stop") - with pytest.raises(asyncio.CancelledError, match="acceptance-stop"): - await asyncio.wait_for(task, timeout=4) - assert owned.closed and local_conn.is_closed() - finally: - if task is not None: - if not task.done(): - task.cancel() - await asyncio.gather(task, return_exceptions=True) - await transaction.rollback() - if observer is not None: - await observer.close() - await blocker.close() + async with contextlib.AsyncExitStack() as cleanup: + try: + blocker = await asyncpg.connect(local_dsn) + cleanup.push_async_callback(blocker.close) + transaction = blocker.transaction() + await transaction.start() + cleanup.push_async_callback(transaction.rollback) + observer = await asyncpg.connect(local_dsn) + cleanup.push_async_callback(observer.close) + await blocker.fetchrow("SELECT id FROM loom.cell_self FOR UPDATE") + task = asyncio.create_task(make().start(owned)) + cleanup.push_async_callback(asyncio.gather, task, return_exceptions=True) + cleanup.callback(task.cancel) + async with asyncio.timeout(4): + while not await observer.fetchval( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 " + "AND wait_event_type='Lock' AND query LIKE 'UPDATE loom.cell_self%')", + backend_pid, + ): + assert not task.done() + await asyncio.sleep(0.005) + task.cancel("acceptance-stop") + with pytest.raises(asyncio.CancelledError, match="acceptance-stop"): + await asyncio.wait_for(task, timeout=4) + assert owned.closed and local_conn.is_closed() + except BaseException: # Every acquired resource must close before failure escapes. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise await _assert_widening_rolled_back(local_dsn, directory_conn, backend_pid, before) @@ -336,43 +336,46 @@ async def delayed(*args, **kwargs): return await real_change(*args, **kwargs) monkeypatch.setattr(directory, "update_configuration", delayed) - pending = asyncio.create_task(make("finish").start(owned)) - admin = await asyncpg.connect(dsn) - successor = await asyncpg.connect(dsn) - await directory._init(successor) - try: - await asyncio.wait_for(entered.wait(), 5) - old_generation = str( - await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") - ) - await admin.execute("SELECT pg_terminate_backend($1)", local_conn.get_server_pid()) - assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) - fresh_owner = owner(successor) - await make().start(fresh_owner) - if aba: - await make("revert").start(fresh_owner) + async with contextlib.AsyncExitStack() as cleanup: + try: + pending = asyncio.create_task(make("finish").start(owned)) + cleanup.push_async_callback(asyncio.gather, pending, return_exceptions=True) + cleanup.callback(pending.cancel) + cleanup.callback(resume.set) + admin = await asyncpg.connect(dsn) + cleanup.push_async_callback(admin.close) + successor = await asyncpg.connect(dsn) + cleanup.push_async_callback(successor.close) + await directory._init(successor) + await asyncio.wait_for(entered.wait(), 5) + old_generation = str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + await admin.execute("SELECT pg_terminate_backend($1)", local_conn.get_server_pid()) + assert await successor.fetchval("SELECT pg_try_advisory_lock($1)", cell.ATTACH_LOCK_KEY) + fresh_owner = owner(successor) await make().start(fresh_owner) - assert old_generation != str( - await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") - ) - assert (await cell_update_pg.read(successor, "first")).pending is not None - resume.set() - with pytest.raises(cell_control.Refused, match="generation|attach lock|lock session"): - await pending - stored = await directory_conn.fetchrow( - "SELECT current_fp,next_fp,placed FROM loom_router.cells" - ) - assert json.loads(stored["current_fp"]) == value() - assert json.loads(stored["next_fp"]) == value("new", orgs=("permanent",)) - assert stored["placed"] == 1 - assert (await cell_update_pg.read(successor, "first")).pending is not None - finally: - resume.set() - if not pending.done(): - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) - await successor.close() - await admin.close() + if aba: + await make("revert").start(fresh_owner) + await make().start(fresh_owner) + assert old_generation != str( + await directory_conn.fetchval("SELECT mutation_generation FROM loom_router.cells") + ) + assert (await cell_update_pg.read(successor, "first")).pending is not None + resume.set() + with pytest.raises(cell_control.Refused, match="generation|attach lock|lock session"): + await pending + stored = await directory_conn.fetchrow( + "SELECT current_fp,next_fp,placed FROM loom_router.cells" + ) + assert json.loads(stored["current_fp"]) == value() + assert json.loads(stored["next_fp"]) == value("new", orgs=("permanent",)) + assert stored["placed"] == 1 + assert (await cell_update_pg.read(successor, "first")).pending is not None + except BaseException: # Owned cleanup must preserve the original test failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise @pytest.mark.asyncio @@ -409,64 +412,70 @@ async def test_update_uses_fresh_real_pg_ch_nonces_and_cleans_only_its_values( client = CHClient() await client.connect() with query_tag("test.cell_update.nonces"): - await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) - try: - await client.execute( - "CREATE TABLE {db:Identifier}.locks (name String, holder String, " - "acquired_at DateTime64(3), renewed_at DateTime64(3), ttl_s UInt32, " - "build_commit String) ENGINE=KeeperMap({path:String}) PRIMARY KEY name", - {"db": database, "path": "/loom/test/" + database}, - ) - monkeypatch.setattr( - leases, "LOCK_RELEASE", leases.LOCK_RELEASE.replace(leases.LOCK_TABLE, table) - ) - monkeypatch.setattr(leases, "LOCK_TABLE", table) - monkeypatch.setattr("server.core.ch.get_client", lambda: client) - monkeypatch.setattr(cell_update_probe, "post", REAL_POST) - monkeypatch.setattr(app.Router, "_stores_check", REAL_STORES_CHECK) - monkeypatch.setattr(cell_prepare, "_sleep", AsyncMock()) - await directory_conn.execute( - "UPDATE loom_router.cells SET probe_url='http://probe',upstream_url='http://main'" - ) - requests = [] - - async def network(request): - if request.url.host == "main": - return httpx.Response(200, json={"deployment_id": "dep-first"}) - requests.append(request) - actual = await cell_probe.nonces() - assert actual["postgres"] and actual["clickhouse"] - if proof == "busy" and len(requests) == 1: - return httpx.Response(429, headers={"Retry-After": "1"}) - return httpx.Response( - 200, - json={ - "deployment_id": "dep-first", - "bucket": {"name": "", "prefix": ""}, - "nonces": actual | ({"postgres": "old"} if proof == "wrong" else {}), - }, + async with contextlib.AsyncExitStack() as cleanup: + cleanup.push_async_callback(client.disconnect) + try: + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + cleanup.push_async_callback( + client.execute, "DROP DATABASE {db:Identifier} SYNC", {"db": database} ) - - router = app.Router(probe_transport=httpx.MockTransport(network)) - run = make() - run.transport = httpx.ASGITransport(app=app.build_app(router)) - run.ch, run.args.wait = client, 3 - if proof == "wrong": - with pytest.raises(cell_prepare.Refused, match="does not read the stores"): - await run.start(owned) - assert await directory_conn.fetchval( - "SELECT next_fp IS NULL FROM loom_router.cells" + await client.execute( + "CREATE TABLE {db:Identifier}.locks (name String, holder String, " + "acquired_at DateTime64(3), renewed_at DateTime64(3), ttl_s UInt32, " + "build_commit String) ENGINE=KeeperMap({path:String}) PRIMARY KEY name", + {"db": database, "path": "/loom/test/" + database}, + ) + monkeypatch.setattr( + leases, "LOCK_RELEASE", leases.LOCK_RELEASE.replace(leases.LOCK_TABLE, table) ) - assert (await cell_update_pg.read(local_conn, "first")).pending is None - else: - await run.start(owned) - assert (await cell_update_pg.read(local_conn, "first")).pending is not None - assert len(requests) == (2 if proof == "busy" else 1) - assert await cell_probe.nonces() == { - "postgres": None, - "clickhouse": None, - "bucket": None, - } - finally: - await client.execute("DROP DATABASE {db:Identifier} SYNC", {"db": database}) - await client.disconnect() + monkeypatch.setattr(leases, "LOCK_TABLE", table) + monkeypatch.setattr("server.core.ch.get_client", lambda: client) + monkeypatch.setattr(cell_update_probe, "post", REAL_POST) + monkeypatch.setattr(app.Router, "_stores_check", REAL_STORES_CHECK) + monkeypatch.setattr(cell_prepare, "_sleep", AsyncMock()) + await directory_conn.execute( + "UPDATE loom_router.cells SET probe_url='http://probe',upstream_url='http://main'" + ) + requests = [] + + async def network(request): + if request.url.host == "main": + return httpx.Response(200, json={"deployment_id": "dep-first"}) + requests.append(request) + actual = await cell_probe.nonces() + assert actual["postgres"] and actual["clickhouse"] + if proof == "busy" and len(requests) == 1: + return httpx.Response(429, headers={"Retry-After": "1"}) + return httpx.Response( + 200, + json={ + "deployment_id": "dep-first", + "bucket": {"name": "", "prefix": ""}, + "nonces": actual | ({"postgres": "old"} if proof == "wrong" else {}), + }, + ) + + router = app.Router(probe_transport=httpx.MockTransport(network)) + run = make() + run.transport = httpx.ASGITransport(app=app.build_app(router)) + run.ch, run.args.wait = client, 3 + if proof == "wrong": + with pytest.raises(cell_prepare.Refused, match="does not read the stores"): + await run.start(owned) + assert await directory_conn.fetchval( + "SELECT next_fp IS NULL FROM loom_router.cells" + ) + assert (await cell_update_pg.read(local_conn, "first")).pending is None + else: + await run.start(owned) + assert (await cell_update_pg.read(local_conn, "first")).pending is not None + assert len(requests) == (2 if proof == "busy" else 1) + assert await cell_probe.nonces() == { + "postgres": None, + "clickhouse": None, + "bucket": None, + } + except BaseException: # Owned cleanup must not replace setup or body failure. + with contextlib.suppress(BaseException): + await cleanup.aclose() + raise diff --git a/tests/unit/test_cell_fingerprint_reservations.py b/tests/unit/test_cell_fingerprint_reservations.py index 37926b479b..461eadd657 100644 --- a/tests/unit/test_cell_fingerprint_reservations.py +++ b/tests/unit/test_cell_fingerprint_reservations.py @@ -110,8 +110,12 @@ def test_candidate_fits_a_live_placement_with_the_largest_org() -> None: from server.router.fingerprints import placement_refusal from server.router.limits import MAX_BODY_BYTES - value = _value(future="") - overhead = len(json.dumps({"org": "", "cell": "first", "fingerprint": value})) + deployment = "11111111-1111-4111-8111-111111111111" + value = _value(future="", deployment_id=deployment) + overhead = max( + len(json.dumps({"org": "", "cell": "first", "fingerprint": value})), + len(json.dumps({"org": "", "deployment_id": deployment, "fingerprint": value})), + ) room = MAX_BODY_BYTES - overhead - 6 * cell.ORG_MAX_BYTES value["fields"]["future"] = "x" * room assert placement_refusal("first", "candidate", value) == "" @@ -119,3 +123,61 @@ def test_candidate_fits_a_live_placement_with_the_largest_org() -> None: reason = placement_refusal("first", "candidate", value) assert "future" in reason assert "x" * 20 not in reason + + +def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: + import json + + import httpx + + from server.router.fingerprints import placement_refusal, readable + from server.router.limits import MAX_BODY_BYTES + + deployment = "11111111-1111-4111-8111-111111111111" + value = _value(deployment_id=deployment, oidc_client_id="") + live = {"org": "", "cell": "a", "fingerprint": value} + room = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * room + org = "\x01" * cell.ORG_MAX_BYTES + assert cell.named_org(org) == org and readable(value) + live["org"] = org + operator = {"org": org, "deployment_id": deployment, "fingerprint": value} + assert ( + len(httpx.Request("POST", "http://router.example.test", json=live).content) + <= MAX_BODY_BYTES + ) + assert ( + len(httpx.Request("POST", "http://router.example.test", json=operator).content) + > MAX_BODY_BYTES + ) + assert "placement body" in placement_refusal("a", "candidate", value) + + +@pytest.mark.parametrize( + "cell_name,deployment", + [("a", "11111111-1111-4111-8111-111111111111"), ("a" * 32, "d")], +) +def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deployment) -> None: + import json + + import httpx + + from server.router.fingerprints import placement_refusal, readable + from server.router.limits import MAX_BODY_BYTES + + value = _value(deployment_id=deployment, oidc_client_id="") + bodies = [ + {"org": "", "cell": cell_name, "fingerprint": value}, + {"org": "", "deployment_id": deployment, "fingerprint": value}, + ] + room = MAX_BODY_BYTES - max(len(json.dumps(body)) for body in bodies) - 6 * cell.ORG_MAX_BYTES + value["fields"]["oidc_client_id"] = "x" * room + org = "\x01" * cell.ORG_MAX_BYTES + assert cell.named_org(org) == org and readable(value) and cell.NAME.fullmatch(cell_name) + for body in bodies: + body["org"] = org + assert ( + len(httpx.Request("POST", "http://router.example.test", json=body).content) + <= MAX_BODY_BYTES + ) + assert placement_refusal(cell_name, "candidate", value) == "" diff --git a/tests/unit/test_code_repo_entry_clone_fields.py b/tests/unit/test_code_repo_entry_clone_fields.py index 795028d8e7..1366d5d555 100644 --- a/tests/unit/test_code_repo_entry_clone_fields.py +++ b/tests/unit/test_code_repo_entry_clone_fields.py @@ -25,7 +25,6 @@ def _configure(monkeypatch: pytest.MonkeyPatch, entries: list[dict]) -> None: monkeypatch.setattr(settings, "code_git_dir", "var/code_repos/clickhouse.git") monkeypatch.setattr(settings, "code_git_remote", "https://github.com/ClickHouse/ClickHouse.git") monkeypatch.setattr(settings, "code_repos_json", json.dumps(entries)) - monkeypatch.setattr(repos, "_publish_config_errors", lambda errors: captured.update(errors)) captured: dict[str, str] = {} @@ -46,7 +45,7 @@ def test_release_branch_of_the_scalar_repo_still_inherits_its_clone( {"namespace": "code-clickhouse-26-1", "git_branch": "26.1"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env(settings, {})} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, captured)} rel = by_ns["code-clickhouse-26-1"] assert rel.git_dir == "var/code_repos/clickhouse.git" assert rel.git_remote == "https://github.com/ClickHouse/ClickHouse.git" @@ -65,7 +64,7 @@ def test_a_different_repo_derives_its_remote_from_repo(monkeypatch: pytest.Monke } ], ) - (cfg,) = repos._configured_from_env(settings, {}) + (cfg,) = repos._configured_from_env(settings, captured) assert cfg.git_remote == "https://github.com/langfuse/langfuse.git" assert cfg.git_dir == "var/code_repos/langfuse.git" assert not captured @@ -83,7 +82,7 @@ def test_an_explicit_remote_is_kept(monkeypatch: pytest.MonkeyPatch) -> None: } ], ) - (cfg,) = repos._configured_from_env(settings, {}) + (cfg,) = repos._configured_from_env(settings, captured) assert cfg.git_remote == "git@github.com:acme/x.git" @@ -97,7 +96,7 @@ def test_a_different_repo_without_git_dir_gets_a_clone_of_its_own( {"namespace": "code-langfuse", "repo": "langfuse/langfuse"}, ], ) - by_ns = {c.namespace: c for c in repos._configured_from_env(settings, {})} + by_ns = {c.namespace: c for c in repos._configured_from_env(settings, captured)} lf = by_ns["code-langfuse"] assert lf.git_dir == "var/code_repos/langfuse.git" assert lf.git_dir != by_ns["code-clickhouse"].git_dir From f14e39eaf728e436ecef7fc8e9da070e5f2d7a2d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 12:58:21 +0000 Subject: [PATCH 170/181] Placement fixtures preserve initialization failures Drain independently registered resources before propagating setup failures; keep teardown errors fatal after successful use. Signed-off-by: zlareb1 --- tests/integration/test_cell_place.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_cell_place.py b/tests/integration/test_cell_place.py index 7024024002..40a1cbdf4a 100644 --- a/tests/integration/test_cell_place.py +++ b/tests/integration/test_cell_place.py @@ -27,7 +27,8 @@ async def stores(monkeypatch): admin = os.environ.get("LOOM_ROUTER_PG_DSN") or pytest.skip("LOOM_ROUTER_PG_DSN not set") name, dsn = await create_database(admin) - async with contextlib.AsyncExitStack() as stack: + stack = contextlib.AsyncExitStack() + try: stack.push_async_callback(drop_database, admin, name) first = await stack.enter_async_context(own_database()) second = await stack.enter_async_context(own_database()) @@ -40,6 +41,8 @@ async def stores(monkeypatch): settings, "router_operator_sha256", hashlib.sha256(OPERATOR.encode()).hexdigest() ) await directory.migrate() + stack.push_async_callback(pg.close_pool) + stack.callback(pg.expect_deployment, None) pg.expect_deployment(None) router_app = app.build_app(app.Router()) real_client = httpx.AsyncClient @@ -49,14 +52,15 @@ async def stores(monkeypatch): lambda **kw: real_client(transport=httpx.ASGITransport(app=router_app), **kw), ) targets = {"first": first, "second": second} - try: - for cell_name, cell_dsn in targets.items(): - await _initialize(cell_dsn, cell_name) - yield targets - finally: - pg.expect_deployment(None) - await pg.close_pool() - await directory.close() + for cell_name, cell_dsn in targets.items(): + await _initialize(cell_dsn, cell_name) + yield targets + except BaseException: # Cleanup must preserve the setup failure. + with contextlib.suppress(BaseException): + await stack.aclose() + raise + else: + await stack.aclose() async def _initialize(dsn, name): From 5c989f4e67eea887998867c477f835c9ac945dc7 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 14:38:06 +0000 Subject: [PATCH 171/181] Cell attach checks fresh SCIM provisioning before serving Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 1 + docs/architecture/cells.md | 20 +- docs/operations/cell-attach.md | 15 +- docs/reference/cli.md | 2 +- server/cli/cell_attach.py | 51 ++- server/cli/cell_attach_beats.py | 4 - server/cli/cell_scim.py | 393 ++++++++++++++++++ .../test_cell_scim_census_live_ch.py | 111 +++++ tests/unit/snapshots/cli_help.json | 4 +- tests/unit/snapshots/cli_tree.json | 16 + tests/unit/test_cell_attach.py | 5 +- tests/unit/test_cell_scim_precondition.py | 298 +++++++++++++ .../test_cell_scim_precondition_failures.py | 93 +++++ 13 files changed, 1000 insertions(+), 13 deletions(-) create mode 100644 server/cli/cell_scim.py create mode 100644 tests/integration/test_cell_scim_census_live_ch.py create mode 100644 tests/unit/test_cell_scim_precondition.py create mode 100644 tests/unit/test_cell_scim_precondition_failures.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8f028af37..49d5a620ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -984,6 +984,7 @@ jobs: tests/integration/test_cell_attach.py tests/integration/test_cell_attach_two_replicas.py tests/integration/test_cell_attach_cleanup.py + tests/integration/test_cell_scim_census_live_ch.py tests/integration/test_cell_control_cleanup.py tests/integration/test_cell_control_fence.py tests/integration/test_directory_control_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 43d489d7b5..69c6d3aac6 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1084,10 +1084,28 @@ refuses on any failure, naming it: revocation prefix, so a person the identity provider provisions later is admitted, since it will never deprovision a person it does not count as assigned. +The provisioning census reads active accounts, current unexpired refresh families and +unrevoked, unexpired personal service accounts in bounded pages of 128 rows and 65,536 +bytes of projected string metadata. It uses each table's actual row key to continue, refuses an oversized +page, and shares one finite scan budget. The same bounded person and ancestry reader +checks every root freshly, including while the cell is recording; an authentication +memo or a deactivation-only check cannot establish this precondition. A current refresh +family supplies only its own stored verified email. Registered machine identities keep +their existing closed exemption; a subject prefix or an absent creator grants nothing. + +With `--revoke-unprovisioned`, each accepted cutoff uses SCIM's own subject lease and +revocation prefix. An outright revocation or a newer cutoff is retained. The command +never creates a provisioned user, changes the operator's revocation row, or revokes a +registered machine identity. An accepted cutoff is reported before confirmation and +audit; a later failure leaves it in force and the census incomplete. Rerun with the same +flag to finish. A dry run and a serving rerun never revoke. + The first attach (`loom cell attach`, its procedure on [attaching the first cell](../operations/cell-attach.md)) turns the existing deployment into the first cell. It aliases nothing (an `oauth_issuer_host` on another host refuses at -`attach/begin`), and refuses with SCIM on until per-cell SCIM ships. A run holds an +`attach/begin`). With SCIM on, it freshly checks the complete provisioning precondition +while still recording, before each mode-commit try, after enforcement and before serving. +A serving rerun repeats that check without changing revocations. A run holds an advisory lock, `cell.ATTACH_LOCK_KEY`, on a session of its own for its life (prepare takes the same lock), after checking that the session is one direct backend that keeps the tenancy marker: a lock left on a pooler's server connection would diff --git a/docs/operations/cell-attach.md b/docs/operations/cell-attach.md index 451de55f3e..83a9a9cdb8 100644 --- a/docs/operations/cell-attach.md +++ b/docs/operations/cell-attach.md @@ -34,7 +34,11 @@ Every item below is a precondition attach checks, or one it cannot check and rel 7. `LOOM_PG_DSN` points straight at Postgres, with no pooler of any mode, for the servers and for attach: attach refuses a session that moves between backends or loses the tenancy marker, and refuses a server whose heartbeat reports `marked` false. -8. SCIM is off: attach refuses with it on. +8. With SCIM on, its tokens belong to this cell and every active account, current + refresh session and personal service-account creator graph satisfies the + [provisioning precondition](../architecture/cells.md#scim-and-sign-in-across-cells). + Provision missing identities, or explicitly cut off their existing tokens with + `--revoke-unprovisioned`. 9. On a ClickHouse cluster of several hosts, attach's user exists on every host, and the cluster has an interserver `` or a `remote_servers` user equal to attach's. One replica that cannot sync blocks attach (`SYSTEM SYNC REPLICA … ON CLUSTER`): an @@ -93,6 +97,15 @@ plus the router timeout, with one extension for a process first seen booting; th is not charged to `--wait`. `--scan-budget` (default 7200 s) bounds each full scan, and step 2 must take at most half of it, since step 4 reads the same tables again. Each Postgres mode or trigger try reserves cleanup time inside its client deadline. + +With SCIM on, `--scan-budget` also bounds each complete provisioning census. It checks +active accounts and their contacts, current refresh families with their stored verified +email, and live personal service accounts with every creator and ancestor. It names an +unprovisioned subject and refuses before serving. `--revoke-unprovisioned` cuts off that +subject's existing tokens under SCIM's revocation prefix; it does not provision a user. +Already accepted cutoffs remain if a later read, audit or subject fails. Rerun the same +command and flag to finish. An outright revocation or a newer cutoff is never narrowed. +`--dry-run --revoke-unprovisioned` writes nothing, and a serving rerun only verifies. On interruption its owner closes the session before cancelling transaction work. A close that cannot complete uses transport abort and reports failure; that abort does not establish that server locks or an unacknowledged commit have disappeared. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3a1b9fb2e3..1c93889b11 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -335,7 +335,7 @@ python scripts/codegen/docs_reference.py | `loom cell which` | Print the lookup's cell and host for an org (the first cell if not placed). | `--org`, `--router` | | `loom cell remove` | Remove the directory row of a cell attaching with no placement. | `--cell`, `--router` | | `loom cell prepare` | Register this cell in the router's directory and write its rows. | `--cell`, `--host`, `--credential-sha256`, `--probe-url`, `--wait`, `--router` | -| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--router` | +| `loom cell attach` | Attach the prepared first cell: place its orgs, serve. | `--cell`, `--wait`, `--scan-budget`, `--dry-run`, `--ch-allow-user`, `--ch-allow-user-directory`, `--ch-allow-object`, `--ch-allow-replica`, `--pg-allow-role`, `--revoke-unprovisioned`, `--router` | | `loom tracker-probe` | Read-only first contact with the tracker source: counts and newest timestamps per upstream table per credential, breaker states, the configured tracker namespaces. | | | `loom tracker-contract-check` | Read-only: does each tracker source's schema meet the record contract (docs/features/tracker/record-contract.md)? | `--source`, `--namespace`, `--json` | | `loom tracker-dupcheck-fit` | Refit tracker.duplicate_check's calibrated confidence for one tracker namespace now: replays the duplicate gold at filing time, fits, stores the fit. | `--namespace`, `--org`, `--negatives`, `--dry-run`, `--out` | diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index e624059924..5d3ae5a558 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -43,7 +43,7 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli import cell_control, cell_scan +from server.cli import cell_control, cell_scan, cell_scim from server.cli._ch import writes from server.cli.cell_ch import Refusal, Refused, read_topology from server.core import cell, cell_triggers @@ -208,8 +208,26 @@ async def check_beats(self, stopped_unset: str) -> beats.Verdict: lambda: beats.read_beats(self.ch), self.judge_beats(stopped_unset), self.wait ) (self.keys_fp,) = verdict.keys_fps + await self.scim_precondition() return verdict + async def scim_precondition(self, *, read_only: bool = False) -> None: + if not cell_scim.required(*self.own_fp): + return + report: dict[str, Any] = {} + self.report["scim"] = report + await cell_scim.check( + self.ch, + self.lock_conn, + self.row, + self.own_fp, + seconds=self.args.scan_budget, + revoke=getattr(self.args, "revoke_unprovisioned", False) and not read_only, + dry_run=self.args.dry_run or read_only, + report=report, + log=self.log, + ) + async def stores_refusals(self) -> list[Refusal]: from server.core import pg_profile @@ -413,6 +431,8 @@ async def ready_for_commit(self) -> list[str]: verdict = self.judge_beats("refuse")(await beats.read_beats(self.ch)) if verdict.refusals: raise Refused(verdict.refusals) + if not verdict.booting: + await self.scim_precondition() return verdict.booting async def commit_mode(self) -> None: @@ -492,6 +512,7 @@ async def _read() -> list[beats.Beat]: verdict = await beats.settle( _read, self.judge_beats("refuse"), floor, extend_s=instances.boot_allowance_s() ) + await self.scim_precondition() if verdict.keys_fps != {self.begun_keys_fp}: raise _refused( "processes", @@ -567,6 +588,29 @@ async def done(self) -> bool: async with pg.dedicated(statement_timeout_ms=_LOCK_STATEMENT_MS) as conn: await cell_control.paired(conn, self.row) state, _ = await apg.read_mode(conn) + own = await conn.fetchrow("SELECT current_fp, next_fp FROM loom.cell_self") + if own is None: + raise _refused("SCIM provisioning", "the accepted local pair is absent") + local_pair = (cell.json_object(own["current_fp"]), cell.json_object(own["next_fp"])) + if not cell.matches({}, local_pair[0]) or ( + own["next_fp"] is not None and not cell.matches({}, local_pair[1]) + ): + raise _refused("SCIM provisioning", "the accepted local pair is unreadable") + if cell_scim.required(*local_pair, self.row.get("current_fp"), self.row.get("next_fp")): + if not cell.matches({}, self.row.get("current_fp")): + raise _refused("SCIM provisioning", "the serving directory pair is unreadable") + report: dict[str, Any] = {} + self.report["scim"] = report + await cell_scim.check( + self.ch, + conn, + self.row, + local_pair, + seconds=self.args.scan_budget, + dry_run=True, + report=report, + log=self.log, + ) if state != "enforced": raise _refused( "postgres cell state", @@ -741,5 +785,10 @@ def add_attach_parser(csub: Any) -> None: ("--pg-allow-role", "A Postgres role that may make an org held (the servers' role)"), ): p.add_argument(flag, action="append", default=[], help=what) + p.add_argument( + "--revoke-unprovisioned", + action="store_true", + help="Cut off pre-attach tokens of unprovisioned subjects; dry-run writes nothing", + ) p.add_argument("--router", help="The router's URL (default: LOOM_CELL_ROUTER_URL)") p.set_defaults(func=cmd_attach) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index f49819a1e0..57cc73bfd1 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -192,10 +192,6 @@ def final_refusals(verdict: Verdict) -> list[Refusal]: "finish the root-secret rollout first", ) ) - if verdict.scim: - out.append( - Refusal("processes", "SCIM is on: attach with SCIM on needs the per-cell SCIM release") - ) return out diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py new file mode 100644 index 0000000000..a2aa1f40ae --- /dev/null +++ b/server/cli/cell_scim.py @@ -0,0 +1,393 @@ +"""Fresh provisioning census before a cell accepts SCIM enforcement.""" + +from __future__ import annotations + +import asyncio +import getpass +import json +import math +import time +from collections.abc import Callable, Mapping +from datetime import UTC, datetime +from typing import Any + +from server.auth import claims, revocation, scim, scim_people, subject_revocation +from server.auth.scim_routes import scim_settings_problem +from server.cli import cell_control +from server.cli.cell_ch import WHOLE_SQL, Refusal, Refused +from server.core import cell, ch +from server.core.datetimes import as_aware_utc +from server.settings import settings + +_PAGE = 128 # not a setting: bounded identity metadata transferred per census read + +# The cursor is the actual FINAL row key, never a subject spelling or OFFSET. +_SOURCES = { + "accounts": ( + "loom.namespaces", + "org", + "namespace", + "owner_email", + "''", + "''", + "toUInt32(0)", + "ifNull(active, true) = true", + ), + "sessions": ( + "loom.oauth_state", + "''", + "id", + "subject", + "verified_email", + "family", + "generation", + "record_type = 'session_refresh' AND expires_at > toUInt32(now())", + ), + "services": ( + "loom.service_accounts", + "org", + "name", + "sa_subject", + "''", + "''", + "toUInt32(0)", + "revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now())", + ), +} + + +def required(*values: Mapping[str, Any] | None) -> bool: + enabled = False + for value in values: + if value is None: + continue + if not cell.matches({}, value): + raise _refuse("an accepted configuration fingerprint is unreadable") + scim_on = value["fields"].get("scim", cell.DEFAULTS["scim"]) + if type(scim_on) is not bool: + raise _refuse("an accepted configuration has an unreadable SCIM policy") + enabled = enabled or scim_on + return enabled + + +def _refuse(reason: str) -> Refused: + return Refused([Refusal("SCIM provisioning", reason)]) + + +def _same(left: Any, right: Any) -> bool: + return json.dumps(left, sort_keys=True, default=str) == json.dumps( + right, sort_keys=True, default=str + ) + + +async def _identity(conn: Any, row: dict[str, Any], own: tuple[Any, Any]) -> dict[str, Any]: + await cell_control.paired(conn, row) + found = await conn.fetchrow( + "SELECT a.deployment_id, f.is_first, f.cell, m.mode, m.since, " + "s.current_fp, s.next_fp, s.updated_at FROM loom.authority a " + "LEFT JOIN loom.cell_first f ON true LEFT JOIN loom.tenancy_mode m ON true " + "LEFT JOIN loom.cell_self s ON true" + ) + if ( + found is None + or found["deployment_id"] != row["deployment_id"] + or found["cell"] != row["cell"] + or type(found["is_first"]) is not bool + or found["is_first"] is not row["is_first"] + or found["mode"] not in ("recording", "enforced") + or (not found["is_first"] and found["mode"] != "enforced") + ): + raise _refuse("the local deployment, first identity or tenancy mode differs") + actual = (cell.json_object(found["current_fp"]), cell.json_object(found["next_fp"])) + if actual[0] is None or not _same(actual, own): + raise _refuse("the accepted local fingerprint pair changed") + if row.get("current_fp") is not None: + reserved = (row["current_fp"], row.get("next_fp")) + if any( + value is not None and not any(_same(value, item) for item in reserved) + for value in actual + ): + raise _refuse("the local fingerprint pair differs from the directory reservation") + elif found["is_first"] is not True: + raise _refuse("a later cell has no acknowledged directory fingerprint pair") + for value in actual: + if ( + value is not None + and value.get("fields", {}).get("deployment_id") != row["deployment_id"] + ): + raise _refuse("an accepted fingerprint names another deployment") + configured = scim.tokens(settings.scim_token) + problem = scim_settings_problem( + settings.scim_token, + settings.oauth_login_provider, + settings.auth_actually_required(), + cell_name=row["cell"], + first=(found["is_first"], found["cell"]), + ) + if not configured or problem: + raise _refuse(problem or "the accepted SCIM policy has no configured token") + return dict(found) + + +def _page_sql(kind: str) -> str: + table, key0, key1, subject, email, family, generation, predicate = _SOURCES[kind] + fields = (key0, key1, "org", subject, email, family) + size = " + ".join(f"length({field})" for field in fields) + bounded = ", ".join(f"if(length({field}) <= {{bytes:UInt32}}, {field}, '')" for field in fields) + return ( + f"SELECT count() AS n, sum(identity_bytes) AS bytes, " + f"if(bytes <= {{bytes:UInt32}}, groupArray({_PAGE})(identity), []) AS identities FROM (" + f"SELECT tuple({bounded}, {generation}, toUnixTimestamp(created_at)) AS identity, " + f"{size} AS identity_bytes FROM {table} FINAL WHERE ({predicate}) AND " + f"({{started:UInt8}} = 0 OR ({key0}, {key1}) > ({{after0:String}}, {{after1:String}})) " + f"ORDER BY {key0}, {key1} LIMIT {_PAGE})" + ) + + +async def _query(client: Any, sql: str, params: dict[str, Any], deadline: float) -> Any: + if scim_people._encoded(params) > scim_people.MAX_BYTES: + raise _refuse("a census cursor exceeds its supported parameter size") + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError("SCIM provisioning census exhausted its scan budget") + seconds = min(left, scim_people._decision_seconds()) + async with asyncio.timeout(seconds): + with ch.statement_timeout(seconds): + return await client.query_one(sql + " " + WHOLE_SQL, params=params) + + +async def _refuses_person(org: str, subject: str, email: str, deadline: float) -> bool: + try: + extra = scim_people._roots(subject, email, (), org) + seconds = min(deadline - time.monotonic(), scim_people._decision_seconds()) + if seconds <= 0: + raise TimeoutError + decision = scim_people._Decision( + org, subject, scim.normalized(email), time.monotonic() + seconds + ) + async with asyncio.timeout(seconds): + return await decision.refuses((subject, *extra)) + except (scim_people._Refused, UnicodeError): + return True + + +async def _current_session(client: Any, identity: tuple[Any, ...], deadline: float) -> bool: + _key0, rid, _org, _subject, _email, family, generation, _created = identity + if not family: + raise _refuse("a refresh grant has no session family") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + async with asyncio.timeout(min(remaining, scim_people._decision_seconds())): + with ch.statement_timeout(min(remaining, scim_people._decision_seconds())): + current = await claims.get(client, "family:" + family) + if current is None: + return False + if current == f"{generation}:{rid}": + return True + parts = current.split(":") + if len(parts) < 2 or not parts[0].isdigit(): + raise _refuse("a refresh session has unreadable current state") + if parts[0] == str(generation) and len(parts) == 4 and parts[1] == "spending": + return True # A put-back or takeover can still restore this grant. + if int(parts[0]) > int(generation) or (len(parts) == 2 and parts[1] == "ended"): + return False + raise _refuse("a refresh grant is ahead of its session state") + + +async def _cutoff( + conn: Any, + org: str, + subject: str, + floor: datetime, + *, + contract: bool, + log: Callable[[str], None], + report: dict[str, Any], +) -> bool: + # The same subject lease serializes deactivation, reactivation and --scim. + async with scim.subject_leases([subject]): + existing = (await revocation.scim_revocations([subject])).get(subject) + if existing is not None and ( + existing["issued_before"] is None or as_aware_utc(existing["issued_before"]) >= floor + ): + return True + if not contract: + return False + if subject_revocation.refusal(subject): + raise _refuse("an unprovisioned account has no revocable subject") + if subject.lower().startswith("sa:"): + name = subject.lower().removeprefix("sa:").partition(":")[0] + registered = await ch.get_client().query_one( + "SELECT active FROM loom.machine_identities FINAL " + "WHERE name = {name:String} LIMIT 1", + params={"name": name}, + ) + if registered is not None: + raise _refuse("a registered machine identity cannot be revoked by SCIM attach") + await cell_control.check_lock(conn) + cutoff = revocation.from_now_on() + if floor > cutoff: + raise _refuse("a stored identity timestamp is ahead of the supported clock-skew bound") + by = "cli:" + getpass.getuser() + written = await revocation.scim_revoke( + subject, + reason="scim: unprovisioned at cell attach", + revoked_by=by, + issued_before=cutoff, + reread=False, + ) + # A later audit/read failure cannot undo this accepted contraction. + report["revoked"] += 1 + log(f"SCIM cutoff accepted for {subject!r}; it remains if a later check fails") + confirmed = (await revocation.scim_revocations([subject])).get(subject) + if confirmed is None or ( + confirmed["issued_before"] is not None + and as_aware_utc(confirmed["issued_before"]) < cutoff + ): + raise _refuse("an accepted SCIM cutoff could not be confirmed; it is not rolled back") + await subject_revocation.audit_revocation( + written, token_id=subject, kind="subject", by=by, org=org, scim=True + ) + await cell_control.check_lock(conn) + return True + + +async def check( + client: Any, + conn: Any, + row: dict[str, Any], + own: tuple[Any, Any], + *, + seconds: float, + revoke: bool = False, + dry_run: bool = False, + new_later: bool = False, + target: dict[str, Any] | None = None, + report: dict[str, Any] | None = None, + log: Callable[[str], None] = lambda _message: None, +) -> None: + """Read the intended policy freshly; contract only under the native attach owner.""" + if not (required(target) if target is not None else required(*own)): + return + if not math.isfinite(seconds) or seconds <= 0: + raise _refuse("the provisioning census needs a finite positive scan budget") + if client is not ch.get_client(): + raise _refuse("the census client differs from this process's configured store") + if target is not None and target.get("fields", {}).get("deployment_id") != row.get( + "deployment_id" + ): + raise _refuse("the intended configuration names another deployment") + status = report if report is not None else {} + status.update(checked=0, revoked=0, unprovisioned=0, complete=False) + deadline = time.monotonic() + seconds + started = datetime.now(UTC) + try: + async with asyncio.timeout(seconds): + with ch.own_write_reads(), ch.query_tag("cli.cell.scim_precondition"): + binding = await _identity(conn, row, own) + if new_later and not binding["is_first"]: + found = await _query( + client, "SELECT count() AS n FROM loom.scim_users FINAL", {}, deadline + ) + if found is None or int(found["n"]): + raise _refuse("a new later cell already holds a SCIM user or tombstone") + floor = revocation.past_every_token_before( + max(as_aware_utc(binding["since"]), as_aware_utc(binding["updated_at"])) + ) + # Recording processes can still mint without positive provisioning. + if binding["mode"] != "enforced" or not required(own[0]): + floor = max(floor, revocation.past_every_token_before(started)) + for kind in _SOURCES: + await _source( + client, + conn, + kind, + floor, + deadline, + contract=revoke and not dry_run, + status=status, + log=log, + ) + after = await _identity(conn, row, own) + if not _same(binding, after): + raise _refuse("local acceptance changed during the provisioning census") + status["complete"] = True + except Refused: + raise + except Exception as error: # Incomplete identity reads never establish provisioning. + raise _refuse( + f"the provisioning census is unavailable ({type(error).__name__}); " + f"{status['revoked']} accepted cutoffs remain; rerun the same command" + ) from error + + +async def _source( + client: Any, + conn: Any, + kind: str, + floor: datetime, + deadline: float, + *, + contract: bool, + status: dict[str, Any], + log: Callable[[str], None], +) -> None: + cursor = ("", "") + started = False + while True: + page = await _query( + client, + _page_sql(kind), + { + "started": int(started), + "after0": cursor[0], + "after1": cursor[1], + "bytes": scim_people.MAX_BYTES, + }, + deadline, + ) + if page is None or int(page["n"]) > _PAGE or int(page["bytes"]) > scim_people.MAX_BYTES: + raise _refuse(f"the {kind} census page exceeds its supported metadata size") + values = sorted(tuple(value) for value in page["identities"]) + if len(values) != int(page["n"]): + raise _refuse(f"the {kind} census page is incomplete") + if not values: + break + for identity in values: + if len(identity) != 8: + raise _refuse(f"the {kind} census row is unreadable") + key0, key1, org, subject, email, _family, _generation, created = identity + key = (key0, key1) + if (started and key <= cursor) or any( + not isinstance(value, str) or "\x00" in value for value in identity[:6] + ): + raise _refuse(f"the {kind} census cursor or identity is unreadable") + cursor, started = key, True + if kind == "sessions" and not await _current_session(client, identity, deadline): + continue + status["checked"] += 1 + if not await _refuses_person(org, subject, email, deadline): + continue + status["unprovisioned"] += 1 + if not subject.strip(): + raise _refuse(f"an active {kind} row has no revocable owner") + row_floor = max( + floor, revocation.past_every_token_before(datetime.fromtimestamp(created, UTC)) + ) + if not await _cutoff( + conn, + org, + subject, + row_floor, + contract=contract, + log=log, + report=status, + ): + raise _refuse( + f"unprovisioned {kind} subject {subject!r} in org {org!r}; " + "provision its complete identity or use --revoke-unprovisioned; " + "a dry run never revokes" + ) + if len(values) < _PAGE: + break diff --git a/tests/integration/test_cell_scim_census_live_ch.py b/tests/integration/test_cell_scim_census_live_ch.py new file mode 100644 index 0000000000..d87d74fc4a --- /dev/null +++ b/tests/integration/test_cell_scim_census_live_ch.py @@ -0,0 +1,111 @@ +"""The provisioning census uses actual bounded ClickHouse and Keeper projections.""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from server.auth import claims +from server.auth.oauth_store import token_id +from server.cli import cell_scim +from tests.integration.test_cell_attach import ( + cell_clickhouse_databases as cell_clickhouse_databases, +) +from tests.integration.test_cell_attach import cell_stores as cell_stores +from tests.integration.test_cell_attach import genuine_cell_processes as genuine_cell_processes +from tests.integration.test_scim_person_live_ch import People +from tests.integration.test_scim_sessions_live_ch import Sessions +from tests.integration.test_scim_sessions_live_ch import sessions as sessions + +pytestmark = pytest.mark.usefixtures("ch_target_required") + + +@pytest.fixture +async def people(cell_stores): + # The existing native owner creates the database pair and removes it at teardown. + # Reuse only the row builders; that owner alone closes its connected client. + yield People(cell_stores["ch"]) + + +async def _page(people: People, kind: str, *, after0="", after1="", bytes_=65536): + return await cell_scim._query( + people.client, + cell_scim._page_sql(kind), + {"started": 1, "after0": after0, "after1": after1, "bytes": bytes_}, + time.monotonic() + 30, + ) + + +async def test_account_census_projects_actual_keys_and_refuses_oversized_transfer(people: People): + owner = people.email("census-owner") + namespace = await people.account(owner, owner, password=True) + page = await _page(people, "accounts", after0=people.org, after1=namespace[:-1]) + identities = [ + tuple(row) for row in page["identities"] if tuple(row[0:2]) == (people.org, namespace) + ] + assert len(identities) == 1 + assert identities[0][:7] == (people.org, namespace, people.org, owner, "", "", 0) + assert int(identities[0][7]) > 0 + assert len(page["identities"]) == page["n"] <= 128 + assert page["bytes"] <= 65536 + + overflow = await _page(people, "accounts", after0=people.org, after1=namespace[:-1], bytes_=1) + assert overflow["n"] > 0 + assert overflow["bytes"] > 1 + assert overflow["identities"] == [] + + +async def test_refresh_census_keeps_verified_email_and_detects_actual_retirement( + sessions: Sessions, +): + person = sessions.people + email = person.email("census-session") + grant = sessions.grant(email=email) + token = uuid.uuid4().hex + rid = token_id(token) + await sessions.save_refresh(token, grant) + page = await _page(person, "sessions", after1=rid[:-1]) + identities = [tuple(row) for row in page["identities"] if row[1] == rid] + assert len(identities) == 1 + identity = identities[0] + assert identity[:7] == ("", rid, person.org, grant["subject"], email, grant["family"], 0) + assert await cell_scim._current_session(person.client, identity, time.monotonic() + 30) + + key = "family:" + grant["family"] + current = await claims.get(person.client, key) + assert current == "0:" + rid + await sessions.retire_refresh({**grant, "held": {key: current}}) + assert not await cell_scim._current_session(person.client, identity, time.monotonic() + 30) + + +async def test_personal_census_uses_the_actual_fresh_creator_graph(people: People): + email = people.email("census-creator") + name = "census-personal-" + uuid.uuid4().hex + subject = "sa:" + name + await people.personal(subject, email) + await people.provision(email) + page = await _page(people, "services", after0=people.org, after1=name[:-1]) + identities = [tuple(row) for row in page["identities"] if tuple(row[0:2]) == (people.org, name)] + assert len(identities) == 1 + assert identities[0][:7] == (people.org, name, people.org, subject, "", "", 0) + assert not await cell_scim._refuses_person(people.org, subject, "", time.monotonic() + 30) + await people.provision(email, active=False) + assert await cell_scim._refuses_person(people.org, subject, "", time.monotonic() + 30) + + +async def test_account_keyset_census_keeps_the_row_after_a_full_page(people: People): + owner = people.email("census-page") + names = [f"census-{index:04}" for index in range(129)] + await people.client.insert( + "loom.namespaces", + [{"org": people.org, "namespace": name, "owner_email": owner} for name in names], + ) + first = await _page(people, "accounts", after0=people.org) + assert first["n"] == 128 + assert {row[1] for row in first["identities"]} == set(names[:128]) + assert {row[0] for row in first["identities"]} == {people.org} + last = max((row[0], row[1]) for row in first["identities"]) + second = await _page(people, "accounts", after0=last[0], after1=last[1]) + assert {row[1] for row in second["identities"] if row[0] == people.org} == {names[128]} diff --git a/tests/unit/snapshots/cli_help.json b/tests/unit/snapshots/cli_help.json index 09c9916da5..0ddbd18af3 100644 --- a/tests/unit/snapshots/cli_help.json +++ b/tests/unit/snapshots/cli_help.json @@ -469,8 +469,8 @@ "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --host HOST The host the cell serves its orgs at --credential-sha256 SHA256 The sha256 of the cell's router credential (`loom cell credential new`) --probe-url PROBE_URL A later cell's probe listener address (not the first's) --wait WAIT Seconds to retry a busy or unreachable probe listener --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom cell attach": [ - "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--router ROUTER]", - "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" + "usage: loom cell attach [-h] --cell CELL [--wait WAIT] [--scan-budget SCAN_BUDGET] [--dry-run] [--ch-allow-user CH_ALLOW_USER] [--ch-allow-user-directory CH_ALLOW_USER_DIRECTORY] [--ch-allow-object CH_ALLOW_OBJECT] [--ch-allow-replica CH_ALLOW_REPLICA] [--pg-allow-role PG_ALLOW_ROLE] [--revoke-unprovisioned] [--router ROUTER]", + "options: -h, --help show this help message and exit --cell CELL This cell's name (LOOM_CELL) --wait WAIT Seconds of wall time the waits share; 0 runs one try of each, with no retry --scan-budget SCAN_BUDGET Seconds one full org scan may take --dry-run Run the refusals and the scan and print what would be placed; write nothing (the router's own begin refusals do not run) --ch-allow-user CH_ALLOW_USER A ClickHouse user that may write the configured databases --ch-allow-user-directory CH_ALLOW_USER_DIRECTORY A ClickHouse user directory the check cannot read --ch-allow-object CH_ALLOW_OBJECT A ClickHouse object (db.name) that writes the databases --ch-allow-replica CH_ALLOW_REPLICA A replica outside the cluster (db.table:replica) --pg-allow-role PG_ALLOW_ROLE A Postgres role that may make an org held (the servers' role) --revoke-unprovisioned Cut off pre-attach tokens of unprovisioned subjects; dry-run writes nothing --router ROUTER The router's URL (default: LOOM_CELL_ROUTER_URL)" ], "loom tracker-probe": [ "usage: loom tracker-probe [-h]", diff --git a/tests/unit/snapshots/cli_tree.json b/tests/unit/snapshots/cli_tree.json index 6d548918fa..794d22cc17 100644 --- a/tests/unit/snapshots/cli_tree.json +++ b/tests/unit/snapshots/cli_tree.json @@ -13123,6 +13123,7 @@ "ch_allow_object", "ch_allow_replica", "pg_allow_role", + "revoke_unprovisioned", "router" ] } @@ -13280,6 +13281,21 @@ "help": "A Postgres role that may make an org held (the servers' role)", "metavar": null }, + { + "option_strings": [ + "--revoke-unprovisioned" + ], + "dest": "revoke_unprovisioned", + "action": "_StoreTrueAction", + "default": false, + "const": true, + "choices": null, + "nargs": 0, + "type": null, + "required": false, + "help": "Cut off pre-attach tokens of unprovisioned subjects; dry-run writes nothing", + "metavar": null + }, { "option_strings": [ "--router" diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index 13978acd55..a289fb9b7f 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -145,9 +145,8 @@ def test_the_final_refusals() -> None: verdict = beats.judge( [_beat("a", value=scim)], "first", (cell.as_value({**FIELDS, "scim": True}),), "refuse" ) - assert [r.reason for r in beats.final_refusals(verdict)] == [ - "SCIM is on: attach with SCIM on needs the per-cell SCIM release" - ] + # Provisioning is checked by the real Run owner after the heartbeat verdict. + assert beats.final_refusals(verdict) == [] two = _judge([_beat("a"), _beat("b", keys_fp="other")]) assert "root-secret fingerprints" in beats.final_refusals(two)[0].reason assert "no live process" in beats.final_refusals(_judge([]))[0].reason diff --git a/tests/unit/test_cell_scim_precondition.py b/tests/unit/test_cell_scim_precondition.py new file mode 100644 index 0000000000..2c0bb0946f --- /dev/null +++ b/tests/unit/test_cell_scim_precondition.py @@ -0,0 +1,298 @@ +"""Attach checks actual provisioning while the first cell is still recording.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from server.auth import authz, revocation, scim, subject_revocation +from server.cli import cell_attach, cell_attach_beats +from server.cli.cell_ch import Refused +from server.core import cell, ch +from server.settings import settings +from tests.unit.test_cell_attach import FIELDS, FakeConn, _args, _beat, _value +from tests.unit.test_scim_people import People + + +class Census(People): + def __init__(self): + super().__init__() + self.sessions = [] + self.services_census = [] + self.revocations = {} + self.writes = [] + self.fail_revocation = None + self.session_states = {} + self.census_reads = [] + + async def query_one(self, sql, params=None): + p = params or {} + if "AS identity_bytes" in sql: + self.census_reads.append(sql) + if self.unavailable: + raise ConnectionError("identity store unavailable") + if "loom.namespaces" in sql: + rows = [ + (org, f"account-{i:05}", org, owner, "", "", 0, 1) + for i, (org, owner, _email, _verified) in enumerate(self.accounts) + ] + elif "loom.oauth_state" in sql: + rows = self.sessions + elif "loom.service_accounts" in sql: + rows = self.services_census + else: + raise AssertionError(sql) + selected = sorted( + row for row in rows if not p["started"] or row[:2] > (p["after0"], p["after1"]) + )[:128] + size = sum(sum(len(value.encode()) for value in row[:6]) for row in selected) + return { + "n": len(selected), + "bytes": size, + "identities": selected if size <= p["bytes"] else [], + } + if "loom.oauth_signin_state" in sql: + value = self.session_states.get(p["k"]) + return {"value": value} if value is not None else None + return await super().query_one(sql, params) + + async def query_all(self, sql, params=None): + if "loom.revoked_tokens" in sql: + return [self.revocations[key] for key in params["keys"] if key in self.revocations] + return await super().query_all(sql, params) + + +@pytest.fixture +def provisioned_run(monkeypatch): + world = Census() + monkeypatch.setattr(settings, "cell", "first") + monkeypatch.setattr(settings, "scim_token", "lsc_first_" + "a" * 64) + monkeypatch.setattr(settings, "oauth_login_provider", "oidc") + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(settings, "identity_provider", "") + monkeypatch.setattr(settings, "document_s3_bucket", "") + monkeypatch.setattr(settings, "document_s3_prefix", "") + monkeypatch.setattr(cell, "mode", lambda: "recording") + monkeypatch.setattr(ch, "get_client", lambda: world) + monkeypatch.setattr(scim, "get_client", lambda: world) + monkeypatch.setattr(authz, "_ENDED", {}) + native = {"mode": "recording", "first": True} + run = cell_attach.Run(world, _args(), SimpleNamespace(), lambda _text: None) + run.row = { + "cell": "first", + "is_first": True, + "deployment_id": "dep-1", + "bucket": "", + "bucket_prefix": "", + } + run.own_fp = (cell.as_value({**FIELDS, "scim": True}), None) + now = datetime.now(UTC) - timedelta(minutes=1) + + def state(): + return { + "deployment_id": "dep-1", + "is_first": native["first"], + "cell": "first", + "mode": native["mode"], + "since": now, + "updated_at": now, + "current_fp": run.own_fp[0], + "next_fp": run.own_fp[1], + } + + run.lock_conn = FakeConn( + [ + ("SELECT deployment_id FROM loom.authority", "dep-1"), + ("SELECT a.deployment_id, f.is_first", lambda: state()), + ] + ) + + async def read_beats(_client): + return [_beat(value={**_value(), **run.own_fp[0]})] + + @asynccontextmanager + async def subject_leases(_subjects): + yield + + async def record(subject, *, issued_before, **_kwargs): + if world.fail_revocation == subject: + raise ConnectionError("revocation store unavailable") + written = { + "jti": revocation.SCIM_PREFIX + subject, + "issued_before": issued_before, + "revoked_by": "cli:operator", + "revoked_at": datetime.now(UTC), + "expires_at": datetime.now(UTC) + timedelta(days=365), + "reason": "scim: unprovisioned at cell attach", + } + world.revocations[written["jti"]] = written + world.writes.append(written) + return written + + monkeypatch.setattr(cell_attach_beats, "read_beats", read_beats) + monkeypatch.setattr(scim, "subject_leases", subject_leases) + monkeypatch.setattr(revocation, "scim_revoke", record) + monkeypatch.setattr(subject_revocation, "audit_revocation", AsyncMock()) + return run, world, native + + +async def _accepted(run): + try: + observed = await run.check_beats("refuse") + except Refused as refused: + observed = refused + assert not isinstance(observed, Refused), str(observed) + return observed + + +async def test_recording_active_person_passes_actual_run(provisioned_run): + run, people, _native = provisioned_run + people.accounts = [("o", "person@example.test", "person@example.test", True)] + people.provision("person@example.test") + verdict = await _accepted(run) + assert verdict.scim is True + assert len(people.census_reads) == 3 + assert people.writes == [] + + +async def test_scim_off_control_preserves_actual_run(provisioned_run, monkeypatch): + run, people, _native = provisioned_run + monkeypatch.setattr(settings, "scim_token", "") + run.own_fp = (cell.as_value({**FIELDS, "scim": False}), None) + people.unavailable = True + assert (await run.check_beats("refuse")).scim is False + assert people.census_reads == [] + assert people.writes == [] + + +async def test_recording_missing_contact_names_actual_owner(provisioned_run): + run, people, _native = provisioned_run + people.accounts = [ + ("o", "person@example.test", "person@example.test", True), + ("o", "person@example.test", "password@example.test", False), + ] + people.provision("person@example.test") + with pytest.raises(Refused) as refused: + await run.check_beats("refuse") + assert "unprovisioned accounts subject 'person@example.test'" in str(refused.value) + assert people.writes == [] + + +async def test_actual_live_refresh_uses_its_stored_verified_email(provisioned_run): + run, people, _native = provisioned_run + people.sessions = [("", "refresh-id", "o", "handle", "person@example.test", "family", 0, 1)] + people.session_states["family:family"] = "0:refresh-id" + people.provision("person@example.test") + assert (await _accepted(run)).scim is True + assert run.report["scim"]["checked"] == 1 + assert people.writes == [] + + +async def test_personal_sa_checks_creator_even_without_inheritance(provisioned_run): + run, people, _native = provisioned_run + people.services_census = [("o", "personal", "o", "sa:personal", "", "", 0, 1)] + people.services["sa:personal"] = {"created_by": "person@example.test", "lineage": []} + people.provision("person@example.test") + assert (await _accepted(run)).scim is True + assert people.writes == [] + + +async def test_actual_registered_bot_remains_exempt(provisioned_run): + run, people, _native = provisioned_run + people.services_census = [("o", "automation", "o", "sa:automation", "", "", 0, 1)] + people.bots["automation"] = True + assert (await _accepted(run)).scim is True + assert people.writes == [] + + +@pytest.mark.parametrize( + "tokens,reason", + [ + ("lsc_other_" + "b" * 64, "names another cell"), + (",".join(["lsc_first_" + "a" * 64] * 3), "at most two"), + ], +) +async def test_actual_run_applies_cell_token_policy(provisioned_run, monkeypatch, tokens, reason): + run, people, _native = provisioned_run + monkeypatch.setattr(settings, "scim_token", tokens) + with pytest.raises(Refused) as refused: + await run.check_beats("refuse") + assert reason in str(refused.value) + assert people.census_reads == [] + assert people.writes == [] + + +async def test_revoke_unprovisioned_contracts_without_provisioning(provisioned_run): + run, people, _native = provisioned_run + run.args.revoke_unprovisioned = True + people.accounts = [("o", "person@example.test", "person@example.test", True)] + assert (await _accepted(run)).scim is True + assert len(people.writes) == 1 + assert people.writes[0]["jti"] == revocation.SCIM_PREFIX + "person@example.test" + assert people.writes[0]["issued_before"] is not None + assert people.users == {} + assert run.report["scim"]["revoked"] == 1 + assert run.report["scim"]["complete"] is True + + +async def test_partial_contraction_retains_prior_accepted_cutoff(provisioned_run): + run, people, _native = provisioned_run + run.args.revoke_unprovisioned = True + people.accounts = [ + ("o", "first@example.test", "first@example.test", True), + ("o", "second@example.test", "second@example.test", True), + ] + people.fail_revocation = "second@example.test" + with pytest.raises(Refused): + await run.check_beats("refuse") + assert len(people.writes) == 1 + assert people.writes[0]["jti"] == revocation.SCIM_PREFIX + "first@example.test" + assert run.report["scim"]["revoked"] == 1 + assert run.report["scim"]["complete"] is False + assert people.users == {} + + +async def test_dry_run_with_revoke_flag_does_not_contract(provisioned_run): + run, people, _native = provisioned_run + run.args.revoke_unprovisioned = True + run.args.dry_run = True + people.accounts = [("o", "person@example.test", "person@example.test", True)] + with pytest.raises(Refused) as refused: + await run.check_beats("refuse") + assert "unprovisioned accounts" in str(refused.value) + assert people.writes == [] + assert people.users == {} + + +@pytest.mark.parametrize("outright", [True, False], ids=["outright", "newer-cutoff"]) +async def test_contraction_never_weakens_existing_scim_row(provisioned_run, outright): + run, people, native = provisioned_run + run.args.revoke_unprovisioned = True + native["mode"] = "enforced" + people.accounts = [("o", "person@example.test", "person@example.test", True)] + key = revocation.SCIM_PREFIX + "person@example.test" + existing = { + "jti": key, + "issued_before": None if outright else datetime.now(UTC) + timedelta(seconds=5), + "revoked_by": "scim", + } + people.revocations[key] = existing.copy() + assert (await _accepted(run)).scim is True + assert people.revocations[key] == existing + assert people.writes == [] + assert run.report["scim"]["revoked"] == 0 + + +async def test_missing_provisioning_does_not_borrow_authentication_memo(provisioned_run): + run, people, _native = provisioned_run + people.accounts = [("o", "person@example.test", "person@example.test", True)] + # Populate the real ordinary helper while recording: deactivation-only allows it. + assert await scim.refuses_subject("person@example.test", org="o") is False + with pytest.raises(Refused) as refused: + await run.check_beats("refuse") + assert "unprovisioned accounts subject" in str(refused.value) + assert people.writes == [] diff --git a/tests/unit/test_cell_scim_precondition_failures.py b/tests/unit/test_cell_scim_precondition_failures.py new file mode 100644 index 0000000000..11f46f0f60 --- /dev/null +++ b/tests/unit/test_cell_scim_precondition_failures.py @@ -0,0 +1,93 @@ +"""A cutoff that landed remains reported when its confirmation fails.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.auth import revocation +from server.auth.revocation import scim_revoke as actual_scim_revoke +from server.cli.cell_ch import Refused +from tests.unit.test_cell_attach_serving_identity import serving_run as serving_run +from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run + + +@pytest.mark.parametrize("confirmation", ["unavailable", "missing", "weaker"]) +async def test_cutoff_insert_is_reported_before_failed_confirmation( + provisioned_run, monkeypatch, confirmation +): + run, people, _native = provisioned_run + run.args.revoke_unprovisioned = True + people.accounts = [("o", "person@example.test", "person@example.test", True)] + read = people.query_all + + async def insert(table, rows): + assert table == "loom.revoked_tokens" + for row in rows: + people.revocations[row["jti"]] = row + people.writes.append(row) + + async def query_all(sql, params=None): + if "loom.revoked_tokens" in sql and people.writes: + if confirmation == "unavailable": + raise ConnectionError("accepted insert; confirmation unavailable") + if confirmation == "missing": + return [] + return [{**people.writes[0], "issued_before": datetime.now(UTC) - timedelta(days=1)}] + return await read(sql, params) + + monkeypatch.setattr(people, "insert", insert, raising=False) + monkeypatch.setattr(people, "query_all", query_all) + monkeypatch.setattr(revocation, "scim_revoke", actual_scim_revoke) + monkeypatch.setattr(revocation, "_next_version", AsyncMock(return_value=7)) + remember = Mock() + monkeypatch.setattr(revocation, "remember_row", remember) + refresh = AsyncMock(side_effect=ConnectionError("full snapshot unavailable")) + monkeypatch.setattr(revocation, "refresh", refresh) + messages = [] + run.log = messages.append + + with pytest.raises(Refused): + await run.check_beats("refuse") + + assert len(people.writes) == 1 + assert people.writes[0]["jti"] == revocation.SCIM_PREFIX + "person@example.test" + assert people.writes[0]["version"] == 7 + assert people.writes[0]["issued_before"] is not None + assert run.report["scim"]["revoked"] == 1 + assert run.report["scim"]["complete"] is False + assert len(messages) == 1 + assert "SCIM cutoff accepted" in messages[0] + assert "remains if a later check fails" in messages[0] + assert remember.call_count == 1 + assert refresh.await_count == 0 + assert people.users == {} + + +@pytest.mark.parametrize("invalid", ["missing-current", "empty-current", "empty-pending"]) +async def test_serving_rerun_refuses_an_unreadable_actual_local_pair(serving_run, invalid): + from tests.unit.test_cell_attach import CURRENT + + run, conn, _router, sessions, mutations = serving_run() + conn.routes.insert( + 0, + ( + "SELECT current_fp, next_fp FROM loom.cell_self", + { + "current_fp": None + if invalid == "missing-current" + else {} + if invalid == "empty-current" + else CURRENT, + "next_fp": {} if invalid == "empty-pending" else None, + }, + ), + ) + with pytest.raises(Refused, match="the accepted local pair is unreadable"): + await run.run() + assert "done" not in run.report + assert sessions == ["opened", "closed"] + assert conn.executed == [] + assert all(call.await_count == 0 for call in mutations) From bdfe9ff95edbc130cbe4218687182d4c159970b5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 14:55:19 +0000 Subject: [PATCH 172/181] SCIM admits provisioned account aliases and ownerless resources Signed-off-by: zlareb1 --- server/auth/oauth.py | 5 +- server/cli/cell_scim.py | 3 + .../test_cell_scim_census_live_ch.py | 10 ++ tests/integration/test_scim_person_live_ch.py | 13 +- tests/unit/test_scim_people.py | 2 +- tests/unit/test_scim_review118_regressions.py | 133 ++++++++++++++++++ tests/unit/test_scim_sessions.py | 12 +- 7 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_scim_review118_regressions.py diff --git a/server/auth/oauth.py b/server/auth/oauth.py index 3f7dc6b75c..73707b1510 100644 --- a/server/auth/oauth.py +++ b/server/auth/oauth.py @@ -798,7 +798,10 @@ async def _account_for( from server.auth.login import SignInUnavailable, resolve_identity_for_email try: - if await scim.refuses_subject(email, org="", verified_email=email): # before JIT writes + # Resolve ownership read-only before borrowing a canonical owner's work alias. + existing = await resolve_identity_for_email(email) + subject, org = (existing[0], existing[1]) if existing is not None else (email, "") + if await scim.refuses_subject(subject, org=org, verified_email=email): return _Denied(_SIGN_IN_REFUSED) await finish_unfinished_account(email) resolved = await resolve_identity_for_email(email) diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py index a2aa1f40ae..a102143bf7 100644 --- a/server/cli/cell_scim.py +++ b/server/cli/cell_scim.py @@ -364,6 +364,9 @@ async def _source( ): raise _refuse(f"the {kind} census cursor or identity is unreadable") cursor, started = key, True + # An ownerless namespace is a resource, not a person's account. + if kind == "accounts" and subject == "": + continue if kind == "sessions" and not await _current_session(client, identity, deadline): continue status["checked"] += 1 diff --git a/tests/integration/test_cell_scim_census_live_ch.py b/tests/integration/test_cell_scim_census_live_ch.py index d87d74fc4a..5b5883ec4b 100644 --- a/tests/integration/test_cell_scim_census_live_ch.py +++ b/tests/integration/test_cell_scim_census_live_ch.py @@ -41,7 +41,17 @@ async def _page(people: People, kind: str, *, after0="", after1="", bytes_=65536 async def test_account_census_projects_actual_keys_and_refuses_oversized_transfer(people: People): owner = people.email("census-owner") namespace = await people.account(owner, owner, password=True) + ownerless = "resource-" + uuid.uuid4().hex + await people.client.insert( + "loom.namespaces", + [{"org": people.org, "namespace": ownerless, "owner_email": "", "active": True}], + ) page = await _page(people, "accounts", after0=people.org, after1=namespace[:-1]) + resources = [ + tuple(row) for row in page["identities"] if tuple(row[0:2]) == (people.org, ownerless) + ] + assert len(resources) == 1 + assert resources[0][:7] == (people.org, ownerless, people.org, "", "", "", 0) identities = [ tuple(row) for row in page["identities"] if tuple(row[0:2]) == (people.org, namespace) ] diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 7ba359657a..34818825b7 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -11,7 +11,7 @@ import pytest -from server.auth import authz, scim, service_accounts +from server.auth import authz, oauth, scim, service_accounts from server.core import cell, ch from server.settings import settings from tests.integration._resources import unique_org @@ -329,3 +329,14 @@ async def test_real_person_queries_report_their_physical_work( assert len(rows) >= 3 assert any(int(row["read_rows"]) > 0 for row in rows) record_property("scim_person_query_log", json.dumps(rows, default=str)) + + +async def test_callback_resolves_verified_work_alias_before_enforcement( + people: People, monkeypatch: pytest.MonkeyPatch +) -> None: + owner, contact = people.email("callback-owner"), people.email("callback-contact") + namespace = await people.account(owner, contact) + await people.provision(owner, work=(contact,)) + monkeypatch.setattr(settings, "oidc_jit_provision", False) + observed = await oauth._account_for(contact, {"kind": "dashboard"}, "unused-upstream") + assert observed == (owner, people.org, [namespace], ["writer"]) diff --git a/tests/unit/test_scim_people.py b/tests/unit/test_scim_people.py index d7e464cb0f..884d271705 100644 --- a/tests/unit/test_scim_people.py +++ b/tests/unit/test_scim_people.py @@ -92,7 +92,7 @@ async def query_one(self, sql, params=None): return { "lineage_count": len(above), "subject_bytes": size, - "inherit_creator_grants": False, + "inherit_creator_grants": row.get("inherit_creator_grants", False), "kind": "personal", "revoked_at": row.get("revoked_at"), "expires_at": row.get("expires_at"), diff --git a/tests/unit/test_scim_review118_regressions.py b/tests/unit/test_scim_review118_regressions.py new file mode 100644 index 0000000000..1df99b7173 --- /dev/null +++ b/tests/unit/test_scim_review118_regressions.py @@ -0,0 +1,133 @@ +"""Provisioning distinguishes resources and resolves verified work aliases read-only.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from server.auth import aliases, jit, login, oauth +from server.cli.cell_ch import Refused +from server.settings import settings +from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run +from tests.unit.test_scim_people import people as people + +OWNER = "owner@example.test" +CONTACT = "contact@example.test" + + +@pytest.mark.parametrize("revoke", [False, True], ids=["check", "explicit-revoke"]) +async def test_ownerless_namespace_is_not_a_person_account(provisioned_run, revoke): + run, world, _native = provisioned_run + run.args.revoke_unprovisioned = revoke + world.accounts = [("o", "", "", True), ("o", OWNER, OWNER, True)] + world.provision(OWNER) + try: + observed = await run.check_beats("refuse") + except Refused as error: + observed = error + assert not isinstance(observed, Refused), str(observed) + assert observed.scim is True + assert run.report["scim"]["checked"] == 1 + assert run.report["scim"]["unprovisioned"] == 0 + assert world.writes == [] + + +async def test_blank_personal_creator_still_refuses(provisioned_run): + run, world, _native = provisioned_run + world.services_census = [("o", "orphan", "o", "sa:orphan", "", "", 0, 1)] + world.services["sa:orphan"] = {"created_by": "", "lineage": []} + with pytest.raises(Refused) as refused: + await run.check_beats("refuse") + assert "unprovisioned services subject 'sa:orphan'" in str(refused.value) + assert world.writes == [] + + +async def test_whitespace_owner_is_not_an_ownerless_namespace(provisioned_run): + run, world, _native = provisioned_run + world.accounts = [("o", " ", "", True)] + with pytest.raises(Refused): + await run.check_beats("refuse") + assert run.report["scim"]["unprovisioned"] == 1 + assert world.writes == [] + + +@pytest.fixture +def callback(people, monkeypatch): + people.accounts = [("acme", OWNER, CONTACT, True)] + reads = people.query_all + lookups = [] + + async def query_all(sql, params=None): + p = params or {} + if "WHERE contact_email = {ident:String} AND active = 1" in sql: + lookups.append(p["ident"]) + return [ + { + "namespace": f"sso-account-{index}", + "owner_email": owner, + "org": org, + "password_hash": "", + "contact_email": email, + } + for index, (org, owner, email, verified) in enumerate(people.accounts) + if email == p["ident"] and verified + ] + if "SELECT owner_email, contact_email" in sql: + return [ + {"owner_email": owner, "contact_email": email} + for org, owner, email, verified in people.accounts + if org == p["org"] and verified and p["sub"] in (owner, email) + ] + if "loom.namespace_grants" in sql: + return [] + return await reads(sql, params) + + monkeypatch.setattr(people, "query_all", query_all) + monkeypatch.setattr(login, "get_client", lambda: people) + monkeypatch.setattr(aliases, "_CACHE", {}) + monkeypatch.setattr(aliases, "_GENERATION", {}) + monkeypatch.setattr(settings, "platform_admins", "") + finish = AsyncMock(return_value=None) + make = AsyncMock(side_effect=AssertionError("the callback must not create an account")) + monkeypatch.setattr(jit, "finish_unfinished_account", finish) + monkeypatch.setattr(jit, "jit_provision_for_email", make) + return people, finish, make, lookups + + +async def test_verified_work_alias_uses_actual_canonical_owner_policy(callback): + world, finish, make, lookups = callback + world.provision(OWNER, work=(CONTACT,)) + observed = await oauth._account_for(CONTACT, {"kind": "dashboard"}, "unused-upstream") + assert observed == (OWNER, "acme", ["sso-account-0"], ["writer"]) + assert lookups + assert finish.await_count == 1 + assert make.await_count == 0 + + +@pytest.mark.parametrize("status", ["unknown", "inactive"]) +async def test_unqualified_callback_never_finishes_or_jits(callback, status): + world, finish, make, _lookups = callback + world.provision(OWNER, work=(CONTACT,)) + email = "unknown@example.test" if status == "unknown" else CONTACT + if status == "inactive": + world.provision(CONTACT, active=False) + finish.side_effect = AssertionError("policy must precede unfinished-account writes") + observed = await oauth._account_for(email, {"kind": "dashboard"}, "unused-upstream") + assert isinstance(observed, oauth._Denied) + assert finish.await_count == 0 + assert make.await_count == 0 + + +async def test_active_primary_contact_preserves_existing_callback(callback): + world, finish, make, _lookups = callback + world.provision(OWNER) + world.provision(CONTACT) + assert await oauth._account_for(CONTACT, {"kind": "dashboard"}, "unused-upstream") == ( + OWNER, + "acme", + ["sso-account-0"], + ["writer"], + ) + assert finish.await_count == 1 + assert make.await_count == 0 diff --git a/tests/unit/test_scim_sessions.py b/tests/unit/test_scim_sessions.py index 80748cf0e5..3b1d741ae7 100644 --- a/tests/unit/test_scim_sessions.py +++ b/tests/unit/test_scim_sessions.py @@ -83,14 +83,16 @@ def grant(**changes): } -async def test_callback_requires_provisioning_before_jit_lookup_or_write( - people, session, monkeypatch -): +async def test_callback_requires_provisioning_before_jit_writes(people, session, monkeypatch): + async def read_identity(email): + return None + async def no_jit(*args, **kwargs): - pytest.fail("an unprovisioned callback must not find or make an account") + pytest.fail("an unprovisioned callback must not finish or create an account") monkeypatch.setattr(jit, "finish_unfinished_account", no_jit) - monkeypatch.setattr(login, "resolve_identity_for_email", no_jit) + monkeypatch.setattr(jit, "jit_provision_for_email", no_jit) + monkeypatch.setattr(login, "resolve_identity_for_email", read_identity) result = await oauth._account_for( "missing@example.test", {"kind": "dashboard"}, "unit-upstream" ) From ca069f12842a523791168c896369ce687c86c9ad Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 15:23:48 +0000 Subject: [PATCH 173/181] SSO refuses ambiguous accounts before JIT repair An unresolved shared contact could repair admin owner grants before refusing sign-in. Check active passwordless and linked accounts before repair or signup while retaining genuine first sign-ins. Signed-off-by: zlareb1 --- server/auth/oauth.py | 6 +- server/auth/sso_links.py | 7 +- tests/unit/test_scim_callback_ambiguity.py | 218 +++++++++++++++++++++ 3 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_scim_callback_ambiguity.py diff --git a/server/auth/oauth.py b/server/auth/oauth.py index 73707b1510..1a8c8103ee 100644 --- a/server/auth/oauth.py +++ b/server/auth/oauth.py @@ -803,13 +803,15 @@ async def _account_for( subject, org = (existing[0], existing[1]) if existing is not None else (email, "") if await scim.refuses_subject(subject, org=org, verified_email=email): return _Denied(_SIGN_IN_REFUSED) + if existing is None and await sso_links.reaching(email): + return _Denied("this email's accounts do not resolve to one account") await finish_unfinished_account(email) resolved = await resolve_identity_for_email(email) if resolved is not None: return resolved if await sso_links.reaching(email): - # Linked accounts that disagree (owners, orgs): fail closed, saying so. - return _Denied("this email's linked accounts do not resolve to one account") + # Reachable accounts that disagree (owners, orgs) fail closed. + return _Denied("this email's accounts do not resolve to one account") if await sso_links.unlinked_accounts(email): # Their password account: proving it links it, no new one is made. return await _offer_link(pending, email, upstream) diff --git a/server/auth/sso_links.py b/server/auth/sso_links.py index 819c6edbab..c144c8cb87 100644 --- a/server/auth/sso_links.py +++ b/server/auth/sso_links.py @@ -173,15 +173,14 @@ async def forget_account(org: str, namespace: str, by: str, *, ch: Any = None) - async def reaching(email: str) -> list[tuple[str, str]]: - """The linked accounts an SSO sign-in with `email` would reach: active, and - still naming the email (a deactivated or released one reaches nothing).""" + """The active accounts SSO reaches: passwordless or linked, still naming the email.""" rows = await get_client().query_all( f""" SELECT org, namespace FROM loom.namespaces FINAL WHERE contact_email = {{email:String}} AND active = 1 - AND (org, namespace) IN ( + AND (password_hash = '' OR (org, namespace) IN ( SELECT org, namespace FROM {TABLE} FINAL WHERE email = {{email:String}} AND linked = 1 - ) + )) """, params={"email": email.strip().lower()}, ) diff --git a/tests/unit/test_scim_callback_ambiguity.py b/tests/unit/test_scim_callback_ambiguity.py new file mode 100644 index 0000000000..7d7bc2f2d7 --- /dev/null +++ b/tests/unit/test_scim_callback_ambiguity.py @@ -0,0 +1,218 @@ +"""Ambiguous callbacks refuse before repairing JIT owner grants.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +from server.auth import aliases, jit, login, oauth, revocation, self_signup, sso_links, tenancy +from server.core import grant_cache, leases +from server.observability import audit +from server.settings import settings +from tests.unit.test_scim_people import people as people + +ORG = "acme" +CONTACT = "contact@example.test" +OWNER = "owner@example.test" +STAMP = datetime(2026, 1, 1, tzinfo=UTC).replace(tzinfo=None) + + +@dataclass +class _Accounts: + rows: list[dict[str, Any]] = field(default_factory=list) + grants: list[dict[str, Any]] = field(default_factory=list) + + def add(self, owner: str, *, org: str = ORG, contact: str = CONTACT) -> None: + self.rows.append( + { + "org": org, + "namespace": f"sso-account-{len(self.rows)}", + "owner_email": owner, + "contact_email": contact, + "password_hash": "", + "active": True, + "tags": [jit.JIT_TAG], + } + ) + + +@pytest.fixture +def callback(people, monkeypatch): + accounts = _Accounts() + read_many, read_one = people.query_all, people.query_one + + async def query_all(sql, params=None): + p = params or {} + if "WHERE contact_email = {ident:String} AND active = 1" in sql: + return [ + dict(r) for r in accounts.rows if r["contact_email"] == p["ident"] and r["active"] + ] + if "toString(owner_email) AS owner" in sql: + return [ + { + "namespace": r["namespace"], + "owner": r["owner_email"], + "active": r["active"], + "tags": r["tags"], + "sso": not r["password_hash"], + } + for r in accounts.rows + if r["org"] == p["org"] and r["contact_email"] == p["ident"] + ] + if "SELECT owner_email, contact_email" in sql: + return [ + dict(r) + for r in accounts.rows + if r["org"] == p["org"] and p["sub"] in (r["owner_email"], r["contact_email"]) + ] + if "SELECT org, namespace FROM loom.namespaces FINAL" in sql: + # These accounts have no explicit SSO links; passwordless rows still qualify. + return [ + {"org": r["org"], "namespace": r["namespace"]} + for r in accounts.rows + if "password_hash = '' OR" in sql + and r["contact_email"] == p["email"] + and r["active"] + and not r["password_hash"] + ] + if "password_hash != ''" in sql or "FROM loom.namespace_grants FINAL" in sql: + return [] + return await read_many(sql, params) + + async def query_one(sql, params=None): + p = params or {} + if "SELECT 1 AS x FROM loom.namespace_grants" in sql: + return next( + ( + {"x": 1} + for r in accounts.grants + if (r["org"], r["namespace"], r["subject"]) == (p["org"], p["ns"], p["sub"]) + ), + None, + ) + if "password_hash = '' AS sso" in sql and "namespace = {ns:String}" in sql: + return next( + ( + {**r, "sso": not r["password_hash"]} + for r in accounts.rows + if (r["org"], r["namespace"]) == (p["org"], p["ns"]) + ), + None, + ) + return await read_one(sql, params) + + async def insert(table, rows, **kwargs): + if table == "loom.namespaces": + accounts.rows.extend(dict(r) for r in rows) + people.accounts.extend( + (r["org"], r["owner_email"], r["contact_email"], not r["password_hash"]) + for r in rows + ) + else: + assert table == "loom.namespace_grants" + accounts.grants.extend(dict(r) for r in rows) + + @asynccontextmanager + async def holding(*args, **kwargs): + yield True + + monkeypatch.setattr(people, "query_all", query_all) + monkeypatch.setattr(people, "query_one", query_one) + monkeypatch.setattr(people, "insert", insert, raising=False) + for module in (jit, login, self_signup, sso_links): + monkeypatch.setattr(module, "get_client", lambda: people) + monkeypatch.setattr(aliases, "_CACHE", {}) + monkeypatch.setattr(aliases, "_GENERATION", {}) + monkeypatch.setattr(settings, "oidc_jit_provision", True) + monkeypatch.setattr(settings, "oidc_jit_org", ORG) + monkeypatch.setattr(settings, "oidc_jit_email_domains", "") + monkeypatch.setattr(settings, "platform_admins", "") + monkeypatch.setattr(self_signup, "naive_utc_now_seconds", lambda: STAMP) + monkeypatch.setattr(self_signup.time, "time_ns", lambda: 17) + monkeypatch.setattr(grant_cache, "changed", AsyncMock()) + monkeypatch.setattr(leases, "holding", holding) + monkeypatch.setattr(revocation, "subjects_revoked", AsyncMock(return_value=False)) + monkeypatch.setattr(tenancy, "ensure_held", AsyncMock()) + monkeypatch.setattr(audit, "emit", AsyncMock()) + return people, accounts + + +def _owner_grant(org: str, namespace: str, owner: str) -> dict[str, Any]: + return { + "org": org, + "namespace": namespace, + "subject": owner, + "role": "admin", + "granted_at": STAMP, + "granted_by": owner, + "revoked_at": None, + "version": 17, + } + + +@pytest.mark.parametrize("ambiguity", ["owners", "orgs", "external-orgs"]) +async def test_ambiguous_callback_never_repairs_owner_grants(callback, ambiguity): + world, accounts = callback + accounts.add(OWNER, org="other" if ambiguity == "external-orgs" else ORG) + accounts.add( + "other@example.test" if ambiguity == "owners" else OWNER, + org=ORG if ambiguity == "owners" else "third" if ambiguity == "external-orgs" else "other", + ) + world.accounts = [(r["org"], r["owner_email"], r["contact_email"], True) for r in accounts.rows] + world.provision(CONTACT) + assert await login.resolve_identity_for_email(CONTACT) is None + observed = await oauth._account_for(CONTACT, {"kind": "dashboard"}, "unused-upstream") + assert accounts.grants == [], ( + f"owner grants written before refusing ambiguity: {accounts.grants}" + ) + assert isinstance(observed, oauth._Denied) + assert len(accounts.rows) == 2 + + +@pytest.mark.parametrize("work_alias", [False, True], ids=["primary", "work-alias"]) +async def test_unique_callback_repairs_its_qualified_owner(callback, work_alias): + world, accounts = callback + contact = CONTACT if work_alias else OWNER + accounts.add(OWNER, contact=contact) + world.accounts = [(ORG, OWNER, contact, True)] + world.provision(OWNER, work=(CONTACT,) if work_alias else ()) + observed = await oauth._account_for(contact, {"kind": "dashboard"}, "unused-upstream") + assert observed == (OWNER, ORG, ["sso-account-0"], ["writer"]) + assert accounts.grants == [_owner_grant(ORG, "sso-account-0", OWNER)] + assert len(accounts.rows) == 1 + + +async def test_absent_provisioned_callback_creates_its_jit_account(callback): + world, accounts = callback + world.provision(CONTACT) + assert await login.resolve_identity_for_email(CONTACT) is None + observed = await oauth._account_for(CONTACT, {"kind": "dashboard"}, "unused-upstream") + assert observed is not None and not isinstance(observed, oauth._Denied) + owner, org, namespaces, roles = observed + assert org == ORG and roles == ["writer"] and namespaces == [owner] + assert accounts.rows == [ + { + "org": ORG, + "namespace": owner, + "owner_email": owner, + "contact_email": CONTACT, + "password_hash": "", + "description": "Auto-provisioned via OIDC SSO.", + "tags": [jit.JIT_TAG], + "active": True, + "conflict_threshold": None, + "retention_days": 0, + "warm_days": 0, + "visibility": "private", + "classification": "standard", + "join_policy": "inherit", + "created_at": STAMP, + "updated_at": STAMP, + } + ] + assert accounts.grants == [_owner_grant(ORG, owner, owner)] From 8dcfd394a83c1464d09c32ac2a87d7bf608ce03f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 15:44:12 +0000 Subject: [PATCH 174/181] Cell control tests bound cancellation and isolate provisioning rows Signed-off-by: zlareb1 --- .../test_cell_update_scim_live_ch.py | 31 +++++++++++++++++++ tests/unit/test_cell_place.py | 19 +++++++++--- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_cell_update_scim_live_ch.py b/tests/integration/test_cell_update_scim_live_ch.py index cdbc758fa6..ea82530bba 100644 --- a/tests/integration/test_cell_update_scim_live_ch.py +++ b/tests/integration/test_cell_update_scim_live_ch.py @@ -2,11 +2,13 @@ from __future__ import annotations +import sys from argparse import Namespace import httpx import pytest +from server.auth import scim from server.cli import cell_control, cell_update_pg from server.cli.cell_update_run import Run from server.core import cell, pg @@ -27,6 +29,35 @@ pytestmark = pytest.mark.usefixtures("ch_target_required") +@pytest.fixture(autouse=True) +async def _clean_people(people: People): + try: + yield people + finally: + first_error = sys.exception() + for sql, params in ( + ( + "ALTER TABLE loom.namespaces DELETE WHERE org = {org:String} " + "SETTINGS mutations_sync = 2", + {"org": people.org}, + ), + ( + "ALTER TABLE loom.scim_users DELETE WHERE id IN {ids:Array(String)} " + "SETTINGS mutations_sync = 2", + {"ids": [scim.user_id(email) for email in people.emails]}, + ), + ): + try: + await people.client.execute(sql, params=params) + except BaseException as error: # noqa: BLE001 — attempt every owned cleanup. + if first_error is None: + first_error = error + else: + first_error.add_note(f"identity cleanup also failed: {type(error).__name__}") + if first_error is not None: + raise first_error + + @pytest.mark.parametrize("case", ["provisioned", "missing-person", "changed-local-pair"]) async def test_update_scim_precondition_uses_actual_acceptance_and_people( cell_stores, people: People, monkeypatch: pytest.MonkeyPatch, case: str diff --git a/tests/unit/test_cell_place.py b/tests/unit/test_cell_place.py index a8536a27b4..026b05383a 100644 --- a/tests/unit/test_cell_place.py +++ b/tests/unit/test_cell_place.py @@ -302,11 +302,20 @@ async def test_local_commit_cancellation_closes_the_session(local): conn, state = local conn.stall_commit = True task = asyncio.create_task(invoke_place()) - await conn.commit_started.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - assert conn.closed and not conn.terminated and not conn.committed + try: + try: + await asyncio.wait_for(conn.commit_started.wait(), 0.5) + except TimeoutError: + if task.done(): + await task + raise + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 0.5) + assert conn.closed and not conn.terminated and not conn.committed + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) async def test_stalled_direct_context_entry_is_bounded(local, monkeypatch): From 6f704fd7d05bab89c131cab60e1aea17518dbb25 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 18:06:04 +0000 Subject: [PATCH 175/181] Participant census rejects conflicts before selecting the newest row Compare repeated identity and version pairs independently of their order so a newer replica row cannot hide conflicting older evidence. Signed-off-by: zlareb1 --- server/cli/cell_update_participants.py | 10 ++--- tests/unit/test_cell_participant_conflicts.py | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_cell_participant_conflicts.py diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index 72a5cb0b37..26a3fa8c73 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -47,6 +47,8 @@ def complete(sql: str, *, sequential: int = 0) -> str: def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: out: dict[str, dict[str, Any]] = {} + # The latest winner cannot hide a conflicting older version from another replica. + seen: dict[tuple[str, int], tuple[str, str]] = {} for row in rows: try: identity = str(UUID(str(row["participant_id"]))) @@ -62,12 +64,10 @@ def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: ): raise cell_control._refused("participant census contains an unreadable row") previous = out.get(identity) - if ( - previous is not None - and int(previous["version"]) == version - and (previous["state"], previous["host_proof"]) != (state, proof) - ): + observed = seen.get((identity, version)) + if observed is not None and observed != (state, proof): raise cell_control._refused(f"participant {identity} has conflicting equal versions") + seen[(identity, version)] = (state, proof) if previous is None or int(previous["version"]) < version: out[identity] = dict(row) return out diff --git a/tests/unit/test_cell_participant_conflicts.py b/tests/unit/test_cell_participant_conflicts.py new file mode 100644 index 0000000000..52883285ae --- /dev/null +++ b/tests/unit/test_cell_participant_conflicts.py @@ -0,0 +1,39 @@ +"""A newer participant row cannot hide disagreeing replicas at an older version.""" + +from __future__ import annotations + +import copy + +import pytest + +from server.cli import cell_control +from server.cli import cell_update_participants as participants +from tests.unit.test_cell_update_participants import participant + +ORDERS = ((0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0)) +PROOF = "host|boot||namespace|1|3|||cell-retirement-v1" + + +@pytest.mark.parametrize("order", ORDERS, ids=lambda order: "".join(map(str, order))) +@pytest.mark.parametrize("difference", ["state", "proof"]) +def test_higher_version_does_not_hide_equal_version_conflict(order, difference): + rows = [participant(1, proof=PROOF, version=2), participant(1, proof=PROOF, version=1)] + conflict = participant(1, proof=PROOF, version=1) + if difference == "state": + conflict["state"] = "retired" + else: + conflict["host_proof"] = PROOF.replace("|1|3|", "|2|3|") + rows.append(conflict) + with pytest.raises(cell_control.Refused, match="conflicting equal versions"): + participants.latest([rows[index] for index in order]) + + +@pytest.mark.parametrize("order", ORDERS, ids=lambda order: "".join(map(str, order))) +def test_matching_old_duplicates_preserve_actual_newest_row_and_input(order): + older = participant(1, proof=PROOF, version=1) + newer = participant(1, "drained", proof=PROOF, version=2) + rows = [newer, older, copy.deepcopy(older)] + ordered = [rows[index] for index in order] + before = copy.deepcopy(ordered) + assert participants.latest(ordered) == {newer["participant_id"]: newer} + assert ordered == before From bf38c57c451e850a04820f247d7084a4b23ecc0d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 18:43:02 +0000 Subject: [PATCH 176/181] Cell checks share topology and catalog helpers Move complete topology, catalog and synchronization implementations below CLI owners so server readers can reuse the same checks. Preserve SQL, refusal behavior and assertion contracts. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- server/cli/cell_attach.py | 9 +- server/cli/cell_attach_beats.py | 2 +- server/cli/cell_attach_pg.py | 2 +- server/cli/cell_control.py | 4 +- server/cli/cell_place.py | 2 +- server/cli/cell_scan.py | 155 +----- server/cli/cell_scim.py | 2 +- server/cli/cell_update_participants.py | 19 +- server/cli/cell_writers.py | 521 +----------------- server/cli/doctor_cell.py | 7 +- server/{cli => core}/cell_ch.py | 2 +- server/core/cell_sync.py | 128 +++++ server/insights/store/cell_catalog.py | 500 +++++++++++++++++ tests/integration/test_cell_attach.py | 8 +- tests/integration/test_cell_scan.py | 20 +- tests/integration/test_cell_unset_gate_pg.py | 2 +- tests/unit/test_cell_attach.py | 6 +- .../unit/test_cell_attach_serving_identity.py | 2 +- tests/unit/test_cell_scan.py | 18 +- tests/unit/test_cell_scim_precondition.py | 2 +- .../test_cell_scim_precondition_failures.py | 2 +- tests/unit/test_cell_update_participants.py | 16 +- tests/unit/test_cell_update_scim_runtime.py | 2 +- tests/unit/test_cell_writers.py | 45 +- tests/unit/test_scim_review118_regressions.py | 2 +- 26 files changed, 763 insertions(+), 717 deletions(-) rename server/{cli => core}/cell_ch.py (99%) create mode 100644 server/core/cell_sync.py create mode 100644 server/insights/store/cell_catalog.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 31a5b39e23..72e465951c 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1682,7 +1682,7 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, objects and replicas outside the operator's allow lists); - `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); + `server/core/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; - `server/auth/tenancy.py` (`ensure_held`): the tenancy check and the placement call, diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index 5d3ae5a558..aff06751cc 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -45,8 +45,9 @@ from server.cli import cell_attach_pg as apg from server.cli import cell_control, cell_scan, cell_scim from server.cli._ch import writes -from server.cli.cell_ch import Refusal, Refused, read_topology -from server.core import cell, cell_triggers +from server.core import cell, cell_sync, cell_triggers +from server.core.cell_ch import Refusal, Refused, read_topology +from server.insights.store import cell_catalog from server.settings import settings _WAIT_S = 600 # not a setting: a flag's default @@ -323,7 +324,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: topology = await read_topology(self.ch) if refusals := topology.refusals(): return refusals, topology, {} - names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + names = await cell_catalog.host_names(self.ch, topology) if topology.multi else {} allowed = cell_writers.Allowed( users=frozenset(self.args.ch_allow_user), user_directories=frozenset(self.args.ch_allow_user_directory), @@ -405,7 +406,7 @@ async def begin(self) -> None: # -- the scans --------------------------------------------------------------------- async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: - budget = cell_scan.Budget(self.args.scan_budget) + budget = cell_sync.Budget(self.args.scan_budget) current, nxt = self.own_fp return await cell_scan.scan( self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index 57cc73bfd1..539e2982f3 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -26,8 +26,8 @@ from typing import Any, Literal from server.cli.cell_attach_pg import Wait -from server.cli.cell_ch import Refusal, Refused from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused StoppedUnset = Literal["refuse", "wait"] diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 89717d5699..0ca96ce4c0 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -18,9 +18,9 @@ from functools import partial from typing import TYPE_CHECKING, Any -from server.cli.cell_ch import Refusal, Refused from server.core import cell, cell_triggers, pg from server.core.cell import ATTACH_LOCK_KEY +from server.core.cell_ch import Refusal, Refused if TYPE_CHECKING: from server.cli.cell_control import Owner diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 93cc7e2e6b..f7f97781c0 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -13,9 +13,9 @@ from uuid import UUID from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal -from server.cli.cell_ch import Refused as Refused from server.core import pg +from server.core.cell_ch import Refusal +from server.core.cell_ch import Refused as Refused from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index 214c79179f..a2f0fdd1c2 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -14,9 +14,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_pg as apg from server.cli._ch import run_connected, writes -from server.cli.cell_ch import Refused from server.cli.cell_control import Owner from server.core import cell, pg +from server.core.cell_ch import Refused from server.router.directory import PLACE_MAX_S from server.settings import settings diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index f66435373f..1f292031b7 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -24,14 +24,13 @@ import asyncio import json import re -import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any -from clickhouse_connect.driver.exceptions import DatabaseError - -from server.cli.cell_ch import ( +from server.core import cell_sync, ch_errors, ch_names, pg +from server.core.cell import ORG_FREE, named_org, org_name_refusal +from server.core.cell_ch import ( TAG, WHOLE, WHOLE_SQL, @@ -42,14 +41,12 @@ per_host, quote, ) -from server.cli.cell_writers import own_refusals -from server.core import ch_errors, ch_names, pg -from server.core.cell import ORG_FREE, named_org, org_name_refusal from server.core.cell_triggers import ORG_TABLES from server.core.ch import query_tag from server.core.ch_errors import error_code, error_line from server.erasure.store.census import classified_tables from server.erasure.store.steps_pg import PG_CENSUS, PG_NO_MEMORY_CONTENT +from server.insights.store.cell_catalog import own_refusals # Tables whose orgs are never placed: rows written before any check (a refused or # unauthenticated request's log row keeps the org it named), and tables a view fills from @@ -91,38 +88,11 @@ # How long past the budget a Postgres try may take before this side gives it up, so the # server's own statement timeout fires first when the server answers. _PG_MARGIN_S = 5.0 # not a setting: a margin over --scan-budget -_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED _UNKNOWN_TABLE_CODES = frozenset({60, 81}) # UNKNOWN_TABLE, UNKNOWN_DATABASE _sleep = asyncio.sleep # the seam the retry tests replace -class Budget: - """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" - - def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: - self.seconds = float(seconds) - self._clock = clock - self._start = clock() - - def elapsed(self) -> float: - return self._clock() - self._start - - def left(self) -> float: - return self.seconds - self.elapsed() - - def require(self, subject: str) -> float: - """The seconds left, or Refused naming `subject` once none are.""" - left = self.left() - if left <= 0: - raise Refused([_out_of_budget(subject, self.seconds)]) - return left - - -def _out_of_budget(subject: str, seconds: float) -> Refusal: - return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") - - def pg_quote(name: str) -> str: return '"' + name.replace('"', '""') + '"' @@ -383,7 +353,7 @@ async def list_tables( ) -> TableList: """The scan's table list in both stores, with every refusal it found, attach's own ClickHouse grants checked first; raises `Refused` when a host answers outside the - cluster's names. `names` is `cell_writers.host_names()` on a cluster of several hosts, + cluster's names. `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty.""" with query_tag(TAG): return await _list_tables(ch, pg_conn, topology, names) @@ -501,7 +471,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: cell_sync.Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -529,7 +499,7 @@ async def pg_orgs(table: str, budget: Budget) -> list[str]: # budget ran out only when there is none left, since the timeout is what was left. timed_out = getattr(failure, "sqlstate", None) == "57014" and budget.left() <= 1 if bound.expired() or timed_out: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None # A connect that finds Postgres down comes as PostgresUnavailableError. if not (pg.is_outage(failure) or isinstance(failure, pg.PostgresUnavailableError)): raise Refused([Refusal(subject, f"the scan failed: {failure}")]) from failure @@ -545,63 +515,6 @@ def _ch_transient(e: BaseException) -> bool: ) -def _on_cluster(topology: Topology) -> str: - return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" - - -async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: - """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries - `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must - answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a - profile's `never_throw` hides none). One that does not finish in time refuses as the - budget when the budget is spent, else with the server's error and the hosts it did not - finish on: a replica that cannot sync.""" - seconds = max(1, int(budget.require(subject))) - # `receive_timeout` bounds SYNC REPLICA's own wait. - bounds = {"max_execution_time": seconds, "receive_timeout": seconds} - what = sql.split(" ON CLUSTER ")[0].split(" `")[0] - try: - if not topology.multi: - await ch.execute(sql, settings=bounds) - return - body = await ch.raw_bytes( - sql, - settings={ - **bounds, - "distributed_ddl_output_mode": "throw", - "distributed_ddl_task_timeout": seconds, - "default_format": "JSONEachRow", - }, - ) - rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") - except Exception as e: # only a timeout is judged here; the rest pass to the caller - if error_code(e) not in _TIMEOUT_CODES: - raise - # The statement was given the budget's whole seconds, so up to one is left over. - if budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} - unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" - raise Refused( - [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] - ) from e - failed = [r for r in rows if int(r["status"]) != 0] - if failed: - raise Refused( - [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] - ) - answered = {str(r.get("host")) for r in rows} - if answered != topology.hosts: - missing = ", ".join(sorted(topology.hosts - answered)) or "none" - raise Refused( - [ - Refusal( - subject, f"the hosts that answered are not the listed ones (missing: {missing})" - ) - ] - ) - - def ch_scan_sql(table: Table, topology: Topology) -> str: """The distinct orgs of `table` as hex (bytes: an org that is not UTF-8 is judged, never decoded into another name); on several hosts, first a row naming the configured host @@ -616,36 +529,6 @@ def ch_scan_sql(table: Table, topology: Topology) -> str: return f"{host} UNION ALL {orgs} FORMAT JSONEachRow" -class BodyError(DatabaseError): - """An error ClickHouse wrote into a response body after it began streaming rows: a - server error as the driver raises one, its text keeping the server's `Code: .`, so it - is judged transient or not, and by code, as any other. `rows` are the rows before it.""" - - def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: - super().__init__(text) - self.rows = list(rows or []) - - -def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: - """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that - is not one (an `{"exception": …}` row, or the text tail a server appends once a - statement fails after its first block), with the rest of the body as its text.""" - lines = list(lines) - rows: list[dict[str, Any]] = [] - for i, line in enumerate(lines): - if not line.strip(): - continue - try: - row = json.loads(line) - except ValueError: - row = None - if not isinstance(row, dict) or key not in row: - exception = row.get("exception") if isinstance(row, dict) else None - raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) - rows.append(row) - return rows - - async def _read_scan( ch: Any, sql: str, params: dict[str, Any], extra: dict[str, Any] ) -> list[dict[str, Any]]: @@ -653,23 +536,25 @@ async def _read_scan( async with ch.raw_stream(sql, params, settings=extra) as stream: async for chunk in stream: chunks.append(chunk) - return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") + return cell_sync._body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs( + ch: Any, table: Table, topology: Topology, budget: cell_sync.Budget +) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) try: # Rows buffered on any replica land before the table is read. - flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{_on_cluster(topology)}" - await _system(ch, flush, topology, budget, subject) + flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{cell_sync._on_cluster(topology)}" + await cell_sync._system(ch, flush, topology, budget, subject) if table.engine.startswith("Replicated"): # Every replica pulls every part acknowledged before now, so the read # through any one of them sees them all. - await _system( + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{_on_cluster(topology)} {table.sql} LIGHTWEIGHT", + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} {table.sql} LIGHTWEIGHT", topology, budget, subject, @@ -695,9 +580,9 @@ async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> if r["k"] == "o" ] # A timeout is the budget only when none is left: a profile's estimate throws early. - if error_code(failure) in _TIMEOUT_CODES and budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - if error_code(failure) in _TIMEOUT_CODES or not _ch_transient(failure): + if error_code(failure) in cell_sync._TIMEOUT_CODES and budget.left() <= 1: + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None + if error_code(failure) in cell_sync._TIMEOUT_CODES or not _ch_transient(failure): raise Refused( [Refusal(subject, f"the scan failed: {error_line(failure)}")] ) from failure @@ -722,7 +607,7 @@ async def scan( ch: Any, tables: TableList, topology: Topology, - budget: Budget, + budget: cell_sync.Budget, configured: Iterable[str] = (), ) -> ScanResult: """The distinct orgs of every table of `tables` and the `configured` orgs; Refused, with diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py index a102143bf7..366f778e52 100644 --- a/server/cli/cell_scim.py +++ b/server/cli/cell_scim.py @@ -14,8 +14,8 @@ from server.auth import claims, revocation, scim, scim_people, subject_revocation from server.auth.scim_routes import scim_settings_problem from server.cli import cell_control -from server.cli.cell_ch import WHOLE_SQL, Refusal, Refused from server.core import cell, ch +from server.core.cell_ch import WHOLE_SQL, Refusal, Refused from server.core.datetimes import as_aware_utc from server.settings import settings diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index 26a3fa8c73..b948d47e8e 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -9,9 +9,9 @@ from typing import Any, cast from uuid import UUID -from server.cli import cell_control, cell_scan, cell_writers -from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology -from server.core import cell, ch_names, instances +from server.cli import cell_control +from server.core import cell, cell_sync, ch_names, instances +from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -19,6 +19,7 @@ participant_tag, proves_stopped, ) +from server.insights.store import cell_catalog from server.insights.store import provider_participants as participants @@ -88,8 +89,8 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: raise cell_control._refused("participant census cannot resolve the configured cluster") if problems := topology.refusals(): raise Refused(problems) - names = await cell_writers.host_names(ch, topology) if topology.multi else {} - if problems := (await cell_writers.own_refusals(ch, topology, names))[0]: + names = await cell_catalog.host_names(ch, topology) if topology.multi else {} + if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: raise Refused(problems) database = ch_names.database() tables = await per_host( @@ -122,14 +123,14 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: ), {"db": database}, ) - if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas( + if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( row for value in rows.values() for row in value ): raise cell_control._refused("participant replica set includes an unverified replica") - budget = cell_scan.Budget(cell_control.TRANSACTION_S) - await cell_scan._system( + budget = cell_sync.Budget(cell_control.TRANSACTION_S) + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} " + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", topology, budget, diff --git a/server/cli/cell_writers.py b/server/cli/cell_writers.py index 0aa0c8e0b8..2d7e6345af 100644 --- a/server/cli/cell_writers.py +++ b/server/cli/cell_writers.py @@ -31,37 +31,15 @@ from dataclasses import dataclass from typing import Any -from server.cli.cell_ch import ( - CLUSTER_SQL, - TAG, - WHOLE_SQL, +from server.core.cell_ch import ( WRITERS_TAG, Refusal, - Refused, Topology, databases, per_host, ) -from server.core import ch_names from server.core.ch import query_tag -from server.core.ch_errors import error_code, error_line -from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME - -_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED - -# Each privilege attach needs and the grants that imply it, closest first. -_IMPLIED_BY: dict[str, tuple[str, ...]] = { - "SELECT": ("ALL",), - "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW TABLES": ("SHOW", "ALL"), - "SHOW COLUMNS": ("SHOW", "ALL"), - "REMOTE": ("SOURCES", "ALL"), - "CLUSTER": ("ALL",), - "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), - "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), -} +from server.insights.store import cell_catalog # Grants that write the configured databases or hand that right on: a user holding one on # them, or globally, must be one the operator names. @@ -163,235 +141,34 @@ class Allowed: replicas: frozenset[str] = frozenset() -@dataclass(frozen=True) -class Grant: - """One privilege a user holds: `database` None for a global grant, `table` None for the - whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial - revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" - - access: str - database: str | None = None - table: str | None = None - grant_option: bool = False - revoked: bool = False - - def covers(self, database: str | None) -> bool: - """Whether this grant reaches all of `database` (None: every database).""" - if self.table is not None: - return False - return self.database is None or (database is not None and _db_match(self, database)) - - def touches(self, database: str) -> bool: - """Whether this grant reaches any table of `database`.""" - return self.database is None or _db_match(self, database) - - -def _db_match(grant: Grant, database: str) -> bool: - assert grant.database is not None - if grant.database.endswith("*"): - return database.startswith(grant.database[:-1]) - return grant.database == database - - -def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: - """Whether `grants` give `access` on all of `database` (None: globally).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) - - -def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: - """The partial revokes in `grants` of `access`, or of a grant implying it, on any part - of `database` (None: anywhere).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return [ - g - for g in grants - if g.revoked - and g.access in wanted - and (database is None or g.database is None or _db_match(g, database)) - ] - - -# -- reading grants ---------------------------------------------------------------------- - -_GRANT = re.compile( - r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " - r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" -) -_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") - - -def _split_privileges(text: str) -> list[str]: - """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" - out, depth, start = [], 0, 0 - for i, ch in enumerate(text): - depth += {"(": 1, ")": -1}.get(ch, 0) - if ch == "," and depth == 0: - out.append(text[start:i]) - start = i + 1 - out.append(text[start:]) - return [p.split("(")[0].strip() for p in out if p.strip()] - - -def _target(on: str) -> tuple[str | None, str | None] | str: - """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a - `READ` grant names (`REMOTE`) as itself.""" - if on == "*": - return (None, None) - parts = [a or b for a, b in _IDENT.findall(on)] - if len(parts) != 2: - return on - db, table = parts - return (None if db == "*" else db, None if table == "*" else table) - - -def parse_show_grants(lines: Iterable[str]) -> list[Grant]: - """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its - roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the - revokes of a grant option alone are left out.""" - out = [] - for line in lines: - m = _GRANT.match(line.strip()) - if m is None or m["privs"].startswith("GRANT OPTION FOR "): - continue - target = _target(m["on"]) - option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" - for access in _split_privileges(m["privs"]): - if isinstance(target, str): - # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it - # as written. - if access in ("READ", "WRITE"): - out.append(Grant(target, grant_option=option, revoked=revoked)) - continue - out.append(Grant(access, target[0], target[1], option, revoked)) - return out - - -def resolve_grants( - user: str, - grant_rows: Iterable[Mapping[str, Any]], - role_rows: Iterable[Mapping[str, Any]], - *, - default_only: bool = False, -) -> list[Grant]: - """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every - role it holds followed (default or not, since it can set any). `default_only` follows - only the user's default roles, the ones a new session has (attach's own reads run in - one), and every role those hold: `granted_role_is_default` marks the user's own grants, - while a role's grant of another always passes it on.""" - # A user and a role may share a name: each holder is keyed by its kind. - granted: dict[tuple[str, str], set[str]] = {} - for r in role_rows: - holder = ( - ("user", str(r["user_name"])) - if r.get("user_name") - else ("role", str(r.get("role_name"))) - ) - if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: - continue - granted.setdefault(holder, set()).add(str(r["granted_role_name"])) - roles: set[str] = set() - todo = list(granted.get(("user", user), ())) - while todo: - role = todo.pop() - if role not in roles: - roles.add(role) - todo.extend(granted.get(("role", role), ())) - out = [] - for r in grant_rows: - if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): - out.append(_row_grant(r)) - return out - - -def _row_grant(r: Mapping[str, Any]) -> Grant: - """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object - `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database - as a prefix.""" - access = str(r["access_type"]) - if access in ("READ", "WRITE") and r.get("access_object"): - access = str(r["access_object"]) - database = None if r.get("database") is None else str(r["database"]) - if database is not None and r.get("is_wildcard") and not database.endswith("*"): - database += "*" - return Grant( - access, - database, - None if r.get("table") is None else str(r["table"]), - bool(r.get("grant_option")), - bool(r.get("is_partial_revoke")), - ) - - # -- judging ----------------------------------------------------------------------------- -def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: - """What attach's own user lacks to read what the check and the scan read.""" - needs: list[tuple[str, str | None]] = [ - ("SELECT", "system"), - ("SHOW USERS", None), - ("SHOW ROLES", None), - ("SHOW ROW POLICIES", None), - ("SHOW TABLES", None), - ("SHOW COLUMNS", None), - ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), - ] - for db in dbs: - needs.append(("SELECT", db)) - if topology.hosts: - needs.append(("SYSTEM SYNC REPLICA", db)) - if topology.multi: - needs += [("REMOTE", None), ("CLUSTER", None)] - out = [] - for access, scope in needs: - if not holds(grants, access, scope): - out.append(_label(access, scope)) - elif revokes := narrowing(grants, access, scope): - # A revoke hides part of what the checks read, which reads short silently. - out.append( - f"{_label(access, scope)} (narrowed by REVOKE " - + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) - + ")" - ) - return out - - -def _label(access: str, db: str | None) -> str: - if access == "REMOTE": - return "READ ON REMOTE" - return f"{access} ON {db}.*" if db else f"{access} ON *.*" - - -def writer_reason(grants: Iterable[Grant], dbs: list[str]) -> str: +def writer_reason(grants: Iterable[cell_catalog.Grant], dbs: list[str]) -> str: """Why a user with `grants` can write the configured databases ('' when it cannot): a grant that writes them from anywhere (`_ANYWHERE_ACCESS`), a writing grant on them or globally, else any grant there it can hand on.""" grants = [g for g in grants if not g.revoked] # a revoke only narrows: ignored for g in grants: if g.access in _ANYWHERE_ACCESS: - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" reaching = [g for g in grants if any(g.touches(db) for db in dbs)] for g in reaching: if g.access in _WRITER_ACCESS or g.access.startswith("ALTER "): - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" for g in reaching: if g.grant_option: - return f"holds {g.access} WITH GRANT OPTION on {_scope(g)}" + return f"holds {g.access} WITH GRANT OPTION on {cell_catalog._scope(g)}" # A row policy created later hides rows from the next scan or check, as a write made # later adds them: on the configured databases or the system tables the checks read. for g in grants: if g.access in _POLICY_ACCESS and (g.touches("system") or any(g.touches(db) for db in dbs)): - return f"holds {g.access} on {_scope(g)}, which can hide rows from the scan" + return ( + f"holds {g.access} on {cell_catalog._scope(g)}, which can hide rows from the scan" + ) return "" -def _scope(g: Grant) -> str: - if g.database is None: - return "*.*" - return f"{g.database}.{g.table or '*'}" - - _TO = re.compile(r"\bTO\s+(?:`((?:[^`\\]|\\.)*)`|(\w+))\s*\.", re.I) _NAME = r"(?:`(?:[^`\\]|\\.)*`|[A-Za-z_]\w*)" # `CREATE … VIEW `: a view's `TO` target follows it, and its name may hold anything. @@ -646,280 +423,19 @@ def _inside_problem( return f"a table of engine {engine}, which reads outside the configured databases" -def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: - """Per replicated table (`db.table`), the replicas Keeper lists that no host of the - cluster is (`replica_is_active` keys against every host's `replica_name`, by - `zookeeper_path`).""" - ours: dict[str, set[str]] = {} - listed: dict[str, set[str]] = {} - names: dict[str, str] = {} - for r in rows: - path = str(r["zookeeper_path"]) - ours.setdefault(path, set()).add(str(r["replica_name"])) - listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) - names.setdefault(path, f"{r['database']}.{r['table']}") - out = {} - for path, keys in listed.items(): - if extra := keys - ours[path]: - out[names[path]] = extra - return out - - # -- the check --------------------------------------------------------------------------- -async def _own_user(ch: Any) -> str: - row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") - return str((row or {}).get("u") or "") - - -async def _show_grants(ch: Any) -> list[Grant]: - """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no - privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" - # Pinned: a profile's `break` could end the list before its REVOKE lines. - rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") - return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) - - -# Per host, each user it defines and that user's grants. -HostGrants = dict[str, dict[str, list[Grant]]] - - -async def host_names(ch: Any, topology: Topology) -> dict[str, str]: - """Each answering host's OS hostname (an opaque group key) and the one configured name - it answers for in the cluster; Refused unless there is one group per listed host, each - with exactly one name, the names together the listing. The topology's own refusals (two - servers under one host name, several shards) come first, then what attach's own user - lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read - through `clusterAllReplicas`.""" - if refusals := topology.refusals(): - raise Refused(refusals) - with query_tag(TAG): - if lacking := await first_refusals(ch, topology): - raise Refused(lacking) - rows = await ch.query_all( - "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " - "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " - f"GROUP BY h {CLUSTER_SQL}", - {"c": topology.cluster}, - ) - where = f"cluster {topology.cluster}" - refusals = [] - out: dict[str, str] = {} - for r in rows: - names = sorted(str(n) for n in r["names"]) - if not names: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host's address in {where} does not resolve to a local interface", - ) - ) - elif len(names) > 1: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host answers for several entries in {where} ({', '.join(names)}): " - "two local entries, or two hosts with one OS hostname", - ) - ) - else: - out[str(r["h"])] = names[0] - named = sorted(out.values()) - if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): - missing = sorted(topology.hosts - set(named)) - refusals.append( - Refusal( - where, - "the answering hosts are not the listed ones" - + (f" (no answer for {', '.join(missing)})" if missing else ""), - ) - ) - if refusals: - raise Refused(refusals) - return out - - -async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: - """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs - no privilege. It prints the grants of every role the user holds, a role a session does - not have included: this first look names what is missing; the rows decide.""" - if missing := own_problems(await _show_grants(ch), topology, databases()): - own = await _own_user(ch) - return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] - return [] - - -async def own_refusals( - ch: Any, topology: Topology, names: Mapping[str, str] -) -> tuple[list[Refusal], HostGrants]: - """What attach's own user lacks to read what the checks read (without it `system.tables` - and `system.columns` would silently list less than there is), and every user's grants - per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several - hosts, that every host runs the reads as this user; then that user's grants as each host - keeps them; then the row policies, which would hide rows from its scan.""" - dbs = databases() - if lacking := await first_refusals(ch, topology): - return lacking, {} - own = await _own_user(ch) - try: - if topology.multi and (wrong := await _process_users(ch, topology, names, own)): - return wrong, {} - per_user, own_grants = await _host_grants(ch, topology, names, own) - out = [] - for host, grants in sorted(own_grants.items()): - # Each host keeps its own users: attach's must hold the same on every one, - # through its default roles alone, as its sessions have them. - if lacks := own_problems(grants, topology, dbs): - out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) - if not out: - out = await _row_policies(ch, topology, names, own, dbs) - except Exception as e: # only a refused read is judged here; the rest pass to the caller - if error_code(e) != _ACCESS_DENIED: - raise - reason = ( - f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " - "held only through a role that is not a default role does not reach its sessions" - ) - return [Refusal(f"user {own}", reason)], {} - return out, per_user - - -async def _row_policies( - ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] -) -> list[Refusal]: - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " - "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " - "WHERE database IN {dbs:Array(String)} OR database = 'system'", - {"dbs": dbs}, - ) - out = [] - for host, found in sorted(rows.items()): - for subject, why in row_policy_problems(found, own): - out.append(Refusal(subject, f"{why}{_on(host)}")) - return out - - -def _as_loom_writes(r: Mapping[str, Any]) -> bool: - """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in - either list could cover or exclude attach's user without naming it.""" - if r["short_name"] == POLICY_NAME: - return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] - return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] - - -def _applies(r: Mapping[str, Any], user: str) -> bool: - if r.get("apply_to_all"): - return user not in (r.get("apply_to_except") or []) - return user in (r.get("apply_to_list") or []) - - -def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: - """What hides rows from attach's own user, per row policy on a table of the configured - databases or a system table. Any policy on a system table refuses: the checks read them. - Once a table has a row policy, a user that no permissive policy covers reads none of its - rows, and a restrictive one filters them: either shortens the scan silently. Loom's own - pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` - letting every other user read every row) while it covers attach's user as written; every - other policy refuses.""" - tables: dict[str, list[Mapping[str, Any]]] = {} - for r in rows: - tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) - out = [] - for table, policies in sorted(tables.items()): - subject = f"row policy on {table}" - if table.startswith("system."): - # The checks read the system tables: a policy there hides what they read. Loom - # writes none. - out += [ - (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") - for p in policies - ] - continue - others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] - for p in policies: - name = str(p["short_name"]) - if name not in (POLICY_NAME, OTHERS_POLICY_NAME): - out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) - elif not _as_loom_writes(p): - out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) - elif name == POLICY_NAME and _applies(p, own): - out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) - if not any( - not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) - for p in others - ): - out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) - return out - - -async def _process_users( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> list[Refusal]: - """On several hosts: each host runs this read as attach's own user, exactly once.""" - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, user FROM {source:processes} " - "WHERE initial_query_id = initialQueryID()", - ) - out = [] - for host, found in sorted(rows.items()): - users = [str(r["user"]) for r in found] - if users != [own]: - out.append( - Refusal( - f"host {host}", - f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " - "interserver secret, or the cluster's user to attach's", - ) - ) - return out - - -async def _host_grants( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> tuple[HostGrants, dict[str, list[Grant]]]: - """Per host, each user it defines and that user's grants, from `system.grants` and - `system.role_grants` on every topology (every granted role followed, default or not); - and per host the grants of `own` through its default roles alone.""" - users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") - # Every column: `access_object` and `is_wildcard` are read where the server has them. - grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") - roles = await per_host( - ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" - ) - every = { - host: { - str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found - } - for host, found in users.items() - } - own_grants = { - host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users - } - return every, own_grants - - -def _on(host: str) -> str: - return f" on host {host}" if host else "" - - async def check_writers( ch: Any, topology: Topology, names: Mapping[str, str], allowed: Allowed ) -> list[Refusal]: """Every ClickHouse writer of the configured databases the operator has not named. - `names` is `host_names()` on a cluster of several hosts, else empty. Raises + `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty. Raises `cell_ch.Refused` when a host answers outside the cluster's names; every other refusal is returned.""" dbs = databases() with query_tag(WRITERS_TAG): - lacking, per_user = await own_refusals(ch, topology, names) + lacking, per_user = await cell_catalog.own_refusals(ch, topology, names) if lacking: return lacking refusals: list[Refusal] = [] @@ -927,7 +443,10 @@ async def check_writers( for user, grants in sorted(users.items()): if user not in allowed.users and (why := writer_reason(grants, dbs)): refusals.append( - Refusal(f"user {user}", f"{why}{_on(host)}: name it with --ch-allow-user") + Refusal( + f"user {user}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-user", + ) ) refusals += await _directories(ch, topology, names, allowed) refusals += await _objects(ch, topology, names, allowed, dbs) @@ -949,7 +468,8 @@ async def _directories( out.append( Refusal( f"user directory {name}", - f"its type, {kind}, defines users this check cannot read{_on(host)}: " + f"its type, {kind}, defines users this check cannot read" + f"{cell_catalog._on(host)}: " "name it with --ch-allow-user-directory", ) ) @@ -980,7 +500,10 @@ async def _objects( for key, why in sorted(object_problems(found, dbs, functions, topology.cluster).items()): if key not in allowed.objects: out.append( - Refusal(f"object {key}", f"{why}{_on(host)}: name it with --ch-allow-object") + Refusal( + f"object {key}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-object", + ) ) return out @@ -998,7 +521,7 @@ async def _replicas( ) every = [r for found in rows.values() for r in found] out = [] - for table, extra in sorted(extra_replicas(every).items()): + for table, extra in sorted(cell_catalog.extra_replicas(every).items()): for replica in sorted(extra): if f"{table}:{replica}" not in allowed.replicas: out.append( diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 292f03f374..194964832c 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -20,6 +20,7 @@ from server.cli._ch import reads from server.cli.doctor import FAIL, PASS, WARN +from server.core import cell_sync # A doctor run never hangs on a cell's stores: each statement, and each session's connect # and reads together. @@ -263,13 +264,13 @@ async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: from server.cli import cell_scan - from server.cli.cell_ch import Refused, databases, read_topology from server.core import pg + from server.core.cell_ch import Refused, databases, read_topology from server.core.cell_triggers import ORG_TABLES topology = await read_topology(ch) if topology.multi: - from server.cli.cell_writers import host_names + from server.insights.store.cell_catalog import host_names names = await host_names(ch, topology) else: @@ -289,7 +290,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] work.sort(key=lambda w: (w[0], str(w[2]))) - budget = cell_scan.Budget(budget_s) + budget = cell_sync.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] refused: list[str] = [] diff --git a/server/cli/cell_ch.py b/server/core/cell_ch.py similarity index 99% rename from server/cli/cell_ch.py rename to server/core/cell_ch.py index 3da3766379..6262a23176 100644 --- a/server/cli/cell_ch.py +++ b/server/core/cell_ch.py @@ -157,7 +157,7 @@ async def per_host( """The rows of `sql`, which selects `hostName() AS h` from `{source:
}`, grouped by the configured name of the host each came from: system table `
` itself on one host (grouped under ''), through `clusterAllReplicas` on several (grouped by `names`, - from `cell_writers.host_names()`).""" + from `cell_catalog.host_names()`).""" m = re.search(r"\{source:(\w+)\}", sql) assert m, sql table = f"system.{m.group(1)}" diff --git a/server/core/cell_sync.py b/server/core/cell_sync.py new file mode 100644 index 0000000000..e723a41784 --- /dev/null +++ b/server/core/cell_sync.py @@ -0,0 +1,128 @@ +"""Complete replica synchronization with the caller's existing scan budget.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable, Iterable +from typing import Any + +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core.cell_ch import Refusal, Refused, Topology, quote +from server.core.ch_errors import error_code, error_line + +_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED + + +class Budget: + """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self._clock = clock + self._start = clock() + + def elapsed(self) -> float: + return self._clock() - self._start + + def left(self) -> float: + return self.seconds - self.elapsed() + + def require(self, subject: str) -> float: + """The seconds left, or Refused naming `subject` once none are.""" + left = self.left() + if left <= 0: + raise Refused([_out_of_budget(subject, self.seconds)]) + return left + + +def _out_of_budget(subject: str, seconds: float) -> Refusal: + return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") + + +def _on_cluster(topology: Topology) -> str: + return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" + + +async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: + """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries + `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must + answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a + profile's `never_throw` hides none). One that does not finish in time refuses as the + budget when the budget is spent, else with the server's error and the hosts it did not + finish on: a replica that cannot sync.""" + seconds = max(1, int(budget.require(subject))) + # `receive_timeout` bounds SYNC REPLICA's own wait. + bounds = {"max_execution_time": seconds, "receive_timeout": seconds} + what = sql.split(" ON CLUSTER ")[0].split(" `")[0] + try: + if not topology.multi: + await ch.execute(sql, settings=bounds) + return + body = await ch.raw_bytes( + sql, + settings={ + **bounds, + "distributed_ddl_output_mode": "throw", + "distributed_ddl_task_timeout": seconds, + "default_format": "JSONEachRow", + }, + ) + rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") + except Exception as e: # only a timeout is judged here; the rest pass to the caller + if error_code(e) not in _TIMEOUT_CODES: + raise + # The statement was given the budget's whole seconds, so up to one is left over. + if budget.left() <= 1: + raise Refused([_out_of_budget(subject, budget.seconds)]) from None + done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} + unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" + raise Refused( + [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] + ) from e + failed = [r for r in rows if int(r["status"]) != 0] + if failed: + raise Refused( + [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] + ) + answered = {str(r.get("host")) for r in rows} + if answered != topology.hosts: + missing = ", ".join(sorted(topology.hosts - answered)) or "none" + raise Refused( + [ + Refusal( + subject, f"the hosts that answered are not the listed ones (missing: {missing})" + ) + ] + ) + + +class BodyError(DatabaseError): + """An error ClickHouse wrote into a response body after it began streaming rows: a + server error as the driver raises one, its text keeping the server's `Code: .`, so it + is judged transient or not, and by code, as any other. `rows` are the rows before it.""" + + def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: + super().__init__(text) + self.rows = list(rows or []) + + +def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: + """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that + is not one (an `{"exception": …}` row, or the text tail a server appends once a + statement fails after its first block), with the rest of the body as its text.""" + lines = list(lines) + rows: list[dict[str, Any]] = [] + for i, line in enumerate(lines): + if not line.strip(): + continue + try: + row = json.loads(line) + except ValueError: + row = None + if not isinstance(row, dict) or key not in row: + exception = row.get("exception") if isinstance(row, dict) else None + raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) + rows.append(row) + return rows diff --git a/server/insights/store/cell_catalog.py b/server/insights/store/cell_catalog.py new file mode 100644 index 0000000000..e973826ae4 --- /dev/null +++ b/server/insights/store/cell_catalog.py @@ -0,0 +1,500 @@ +"""Complete catalog visibility, replica identity and grant checks for cell reads.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from server.core import ch_names +from server.core.cell_ch import ( + CLUSTER_SQL, + TAG, + WHOLE_SQL, + Refusal, + Refused, + Topology, + databases, + per_host, +) +from server.core.ch import query_tag +from server.core.ch_errors import error_code, error_line +from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME + +_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED + +_IMPLIED_BY: dict[str, tuple[str, ...]] = { + "SELECT": ("ALL",), + "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW TABLES": ("SHOW", "ALL"), + "SHOW COLUMNS": ("SHOW", "ALL"), + "REMOTE": ("SOURCES", "ALL"), + "CLUSTER": ("ALL",), + "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), + "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), +} + + +@dataclass(frozen=True) +class Grant: + """One privilege a user holds: `database` None for a global grant, `table` None for the + whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial + revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" + + access: str + database: str | None = None + table: str | None = None + grant_option: bool = False + revoked: bool = False + + def covers(self, database: str | None) -> bool: + """Whether this grant reaches all of `database` (None: every database).""" + if self.table is not None: + return False + return self.database is None or (database is not None and _db_match(self, database)) + + def touches(self, database: str) -> bool: + """Whether this grant reaches any table of `database`.""" + return self.database is None or _db_match(self, database) + + +def _db_match(grant: Grant, database: str) -> bool: + assert grant.database is not None + if grant.database.endswith("*"): + return database.startswith(grant.database[:-1]) + return grant.database == database + + +def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: + """Whether `grants` give `access` on all of `database` (None: globally).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) + + +def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: + """The partial revokes in `grants` of `access`, or of a grant implying it, on any part + of `database` (None: anywhere).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return [ + g + for g in grants + if g.revoked + and g.access in wanted + and (database is None or g.database is None or _db_match(g, database)) + ] + + +_GRANT = re.compile( + r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " + r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" +) + +_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") + + +def _split_privileges(text: str) -> list[str]: + """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" + out, depth, start = [], 0, 0 + for i, ch in enumerate(text): + depth += {"(": 1, ")": -1}.get(ch, 0) + if ch == "," and depth == 0: + out.append(text[start:i]) + start = i + 1 + out.append(text[start:]) + return [p.split("(")[0].strip() for p in out if p.strip()] + + +def _target(on: str) -> tuple[str | None, str | None] | str: + """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a + `READ` grant names (`REMOTE`) as itself.""" + if on == "*": + return (None, None) + parts = [a or b for a, b in _IDENT.findall(on)] + if len(parts) != 2: + return on + db, table = parts + return (None if db == "*" else db, None if table == "*" else table) + + +def parse_show_grants(lines: Iterable[str]) -> list[Grant]: + """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its + roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the + revokes of a grant option alone are left out.""" + out = [] + for line in lines: + m = _GRANT.match(line.strip()) + if m is None or m["privs"].startswith("GRANT OPTION FOR "): + continue + target = _target(m["on"]) + option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" + for access in _split_privileges(m["privs"]): + if isinstance(target, str): + # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it + # as written. + if access in ("READ", "WRITE"): + out.append(Grant(target, grant_option=option, revoked=revoked)) + continue + out.append(Grant(access, target[0], target[1], option, revoked)) + return out + + +def resolve_grants( + user: str, + grant_rows: Iterable[Mapping[str, Any]], + role_rows: Iterable[Mapping[str, Any]], + *, + default_only: bool = False, +) -> list[Grant]: + """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every + role it holds followed (default or not, since it can set any). `default_only` follows + only the user's default roles, the ones a new session has (attach's own reads run in + one), and every role those hold: `granted_role_is_default` marks the user's own grants, + while a role's grant of another always passes it on.""" + # A user and a role may share a name: each holder is keyed by its kind. + granted: dict[tuple[str, str], set[str]] = {} + for r in role_rows: + holder = ( + ("user", str(r["user_name"])) + if r.get("user_name") + else ("role", str(r.get("role_name"))) + ) + if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: + continue + granted.setdefault(holder, set()).add(str(r["granted_role_name"])) + roles: set[str] = set() + todo = list(granted.get(("user", user), ())) + while todo: + role = todo.pop() + if role not in roles: + roles.add(role) + todo.extend(granted.get(("role", role), ())) + out = [] + for r in grant_rows: + if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): + out.append(_row_grant(r)) + return out + + +def _row_grant(r: Mapping[str, Any]) -> Grant: + """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object + `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database + as a prefix.""" + access = str(r["access_type"]) + if access in ("READ", "WRITE") and r.get("access_object"): + access = str(r["access_object"]) + database = None if r.get("database") is None else str(r["database"]) + if database is not None and r.get("is_wildcard") and not database.endswith("*"): + database += "*" + return Grant( + access, + database, + None if r.get("table") is None else str(r["table"]), + bool(r.get("grant_option")), + bool(r.get("is_partial_revoke")), + ) + + +def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: + """What attach's own user lacks to read what the check and the scan read.""" + needs: list[tuple[str, str | None]] = [ + ("SELECT", "system"), + ("SHOW USERS", None), + ("SHOW ROLES", None), + ("SHOW ROW POLICIES", None), + ("SHOW TABLES", None), + ("SHOW COLUMNS", None), + ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), + ] + for db in dbs: + needs.append(("SELECT", db)) + if topology.hosts: + needs.append(("SYSTEM SYNC REPLICA", db)) + if topology.multi: + needs += [("REMOTE", None), ("CLUSTER", None)] + out = [] + for access, scope in needs: + if not holds(grants, access, scope): + out.append(_label(access, scope)) + elif revokes := narrowing(grants, access, scope): + # A revoke hides part of what the checks read, which reads short silently. + out.append( + f"{_label(access, scope)} (narrowed by REVOKE " + + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) + + ")" + ) + return out + + +def _label(access: str, db: str | None) -> str: + if access == "REMOTE": + return "READ ON REMOTE" + return f"{access} ON {db}.*" if db else f"{access} ON *.*" + + +def _scope(g: Grant) -> str: + if g.database is None: + return "*.*" + return f"{g.database}.{g.table or '*'}" + + +def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: + """Per replicated table (`db.table`), the replicas Keeper lists that no host of the + cluster is (`replica_is_active` keys against every host's `replica_name`, by + `zookeeper_path`).""" + ours: dict[str, set[str]] = {} + listed: dict[str, set[str]] = {} + names: dict[str, str] = {} + for r in rows: + path = str(r["zookeeper_path"]) + ours.setdefault(path, set()).add(str(r["replica_name"])) + listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) + names.setdefault(path, f"{r['database']}.{r['table']}") + out = {} + for path, keys in listed.items(): + if extra := keys - ours[path]: + out[names[path]] = extra + return out + + +async def _own_user(ch: Any) -> str: + row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") + return str((row or {}).get("u") or "") + + +async def _show_grants(ch: Any) -> list[Grant]: + """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no + privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" + # Pinned: a profile's `break` could end the list before its REVOKE lines. + rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") + return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) + + +HostGrants = dict[str, dict[str, list[Grant]]] + + +async def host_names(ch: Any, topology: Topology) -> dict[str, str]: + """Each answering host's OS hostname (an opaque group key) and the one configured name + it answers for in the cluster; Refused unless there is one group per listed host, each + with exactly one name, the names together the listing. The topology's own refusals (two + servers under one host name, several shards) come first, then what attach's own user + lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read + through `clusterAllReplicas`.""" + if refusals := topology.refusals(): + raise Refused(refusals) + with query_tag(TAG): + if lacking := await first_refusals(ch, topology): + raise Refused(lacking) + rows = await ch.query_all( + "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " + "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " + f"GROUP BY h {CLUSTER_SQL}", + {"c": topology.cluster}, + ) + where = f"cluster {topology.cluster}" + refusals = [] + out: dict[str, str] = {} + for r in rows: + names = sorted(str(n) for n in r["names"]) + if not names: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host's address in {where} does not resolve to a local interface", + ) + ) + elif len(names) > 1: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host answers for several entries in {where} ({', '.join(names)}): " + "two local entries, or two hosts with one OS hostname", + ) + ) + else: + out[str(r["h"])] = names[0] + named = sorted(out.values()) + if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): + missing = sorted(topology.hosts - set(named)) + refusals.append( + Refusal( + where, + "the answering hosts are not the listed ones" + + (f" (no answer for {', '.join(missing)})" if missing else ""), + ) + ) + if refusals: + raise Refused(refusals) + return out + + +async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: + """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs + no privilege. It prints the grants of every role the user holds, a role a session does + not have included: this first look names what is missing; the rows decide.""" + if missing := own_problems(await _show_grants(ch), topology, databases()): + own = await _own_user(ch) + return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] + return [] + + +async def own_refusals( + ch: Any, topology: Topology, names: Mapping[str, str] +) -> tuple[list[Refusal], HostGrants]: + """What attach's own user lacks to read what the checks read (without it `system.tables` + and `system.columns` would silently list less than there is), and every user's grants + per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several + hosts, that every host runs the reads as this user; then that user's grants as each host + keeps them; then the row policies, which would hide rows from its scan.""" + dbs = databases() + if lacking := await first_refusals(ch, topology): + return lacking, {} + own = await _own_user(ch) + try: + if topology.multi and (wrong := await _process_users(ch, topology, names, own)): + return wrong, {} + per_user, own_grants = await _host_grants(ch, topology, names, own) + out = [] + for host, grants in sorted(own_grants.items()): + # Each host keeps its own users: attach's must hold the same on every one, + # through its default roles alone, as its sessions have them. + if lacks := own_problems(grants, topology, dbs): + out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) + if not out: + out = await _row_policies(ch, topology, names, own, dbs) + except Exception as e: # only a refused read is judged here; the rest pass to the caller + if error_code(e) != _ACCESS_DENIED: + raise + reason = ( + f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " + "held only through a role that is not a default role does not reach its sessions" + ) + return [Refusal(f"user {own}", reason)], {} + return out, per_user + + +async def _row_policies( + ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] +) -> list[Refusal]: + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " + "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " + "WHERE database IN {dbs:Array(String)} OR database = 'system'", + {"dbs": dbs}, + ) + out = [] + for host, found in sorted(rows.items()): + for subject, why in row_policy_problems(found, own): + out.append(Refusal(subject, f"{why}{_on(host)}")) + return out + + +def _as_loom_writes(r: Mapping[str, Any]) -> bool: + """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in + either list could cover or exclude attach's user without naming it.""" + if r["short_name"] == POLICY_NAME: + return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] + return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] + + +def _applies(r: Mapping[str, Any], user: str) -> bool: + if r.get("apply_to_all"): + return user not in (r.get("apply_to_except") or []) + return user in (r.get("apply_to_list") or []) + + +def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: + """What hides rows from attach's own user, per row policy on a table of the configured + databases or a system table. Any policy on a system table refuses: the checks read them. + Once a table has a row policy, a user that no permissive policy covers reads none of its + rows, and a restrictive one filters them: either shortens the scan silently. Loom's own + pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` + letting every other user read every row) while it covers attach's user as written; every + other policy refuses.""" + tables: dict[str, list[Mapping[str, Any]]] = {} + for r in rows: + tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) + out = [] + for table, policies in sorted(tables.items()): + subject = f"row policy on {table}" + if table.startswith("system."): + # The checks read the system tables: a policy there hides what they read. Loom + # writes none. + out += [ + (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") + for p in policies + ] + continue + others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] + for p in policies: + name = str(p["short_name"]) + if name not in (POLICY_NAME, OTHERS_POLICY_NAME): + out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) + elif not _as_loom_writes(p): + out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) + elif name == POLICY_NAME and _applies(p, own): + out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) + if not any( + not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) + for p in others + ): + out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) + return out + + +async def _process_users( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> list[Refusal]: + """On several hosts: each host runs this read as attach's own user, exactly once.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, user FROM {source:processes} " + "WHERE initial_query_id = initialQueryID()", + ) + out = [] + for host, found in sorted(rows.items()): + users = [str(r["user"]) for r in found] + if users != [own]: + out.append( + Refusal( + f"host {host}", + f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " + "interserver secret, or the cluster's user to attach's", + ) + ) + return out + + +async def _host_grants( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> tuple[HostGrants, dict[str, list[Grant]]]: + """Per host, each user it defines and that user's grants, from `system.grants` and + `system.role_grants` on every topology (every granted role followed, default or not); + and per host the grants of `own` through its default roles alone.""" + users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") + # Every column: `access_object` and `is_wildcard` are read where the server has them. + grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") + roles = await per_host( + ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" + ) + every = { + host: { + str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found + } + for host, found in users.items() + } + own_grants = { + host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users + } + return every, own_grants + + +def _on(host: str) -> str: + return f" on host {host}" if host else "" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 72f6604578..b621fd61d9 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -29,9 +29,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach, cell_attach_pg, cell_prepare -from server.cli.cell_ch import Refused -from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core import cell, cell_probe, cell_sync, cell_triggers, instances, pg from server.core import ch as ch_core +from server.core.cell_ch import Refused from server.core.schema import pg_migrate from server.router import app as router_app from server.router import directory @@ -895,9 +895,9 @@ async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) clock = [0.0] - real_budget = cell_scan.Budget + real_budget = cell_sync.Budget monkeypatch.setattr( - cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + cell_sync, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) ) # The budget runs out on the largest table, as it does on a real cell when that table's # read is what is too slow: every smaller one was read first. diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index c78bc40331..32a9560c6f 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -17,11 +17,13 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused -from server.cli.cell_scan import Budget, Table, TableList -from server.core import cell, ch_names, pg +from server.cli import cell_scan, cell_writers +from server.cli.cell_scan import Table, TableList +from server.core import cell, cell_ch, ch_names, pg +from server.core.cell_ch import Refused +from server.core.cell_sync import Budget from server.core.ch import get_client +from server.insights.store import cell_catalog from server.settings import settings from tests.integration._pg_database import own_database from tests.integration._resources import unique_org @@ -287,12 +289,12 @@ async def test_grants_are_read_as_the_server_keeps_them( await ch.execute(f"GRANT {role} TO {users['writer']}") await ch.execute(f"SET DEFAULT ROLE NONE TO {users['writer']}") topology = await cell_ch.read_topology(ch) - every, _ = await cell_writers._host_grants(ch, topology, {}, "") + every, _ = await cell_catalog._host_grants(ch, topology, {}, "") (grants,) = every.values() - assert f"SELECT ON {schema}.*" in cell_writers.own_problems( + assert f"SELECT ON {schema}.*" in cell_catalog.own_problems( grants[users["narrow"]], topology, [schema] ) - assert cell_writers.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] + assert cell_catalog.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] assert cell_writers.writer_reason(grants[users["writer"]], [schema]) == ( f"holds INSERT on {schema}.*" ) @@ -516,7 +518,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai created.remove(theirs) # Loom's own pair: the analyst's scope, and every other user reading every row, with # a scratch user standing in for the analyst. - monkeypatch.setattr(cell_writers, "ANALYST_USER", other) + monkeypatch.setattr(cell_catalog, "ANALYST_USER", other) for name, using, to in ( (POLICY_NAME, "org = ''", other), (OTHERS_POLICY_NAME, "1", f"ALL EXCEPT {other}"), @@ -527,7 +529,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai ) created.append(name) counted = await ch.query_one(f"SELECT count() AS n FROM {schema}.memory") - refusals = await cell_writers._row_policies(ch, topology, {}, own, cell_ch.databases()) + refusals = await cell_catalog._row_policies(ch, topology, {}, own, cell_ch.databases()) finally: for name in created: await ch.execute(f"DROP ROW POLICY IF EXISTS `{name}` ON {schema}.memory") diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index 37e80a7057..e2ab59c65b 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -251,7 +251,7 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( import httpx from server.cli import cell as cell_cli - from server.cli.cell_ch import Refused + from server.core.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index a289fb9b7f..6862f8ea0d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -18,9 +18,9 @@ from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal, Refused from server.cli.cell_scan import Table from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings from tests.unit.cell_control_owner import owner as _owner @@ -1443,8 +1443,8 @@ async def test_the_stray_read_names_a_refused_table_and_reads_on( ) -> None: from contextlib import asynccontextmanager - from server.cli import cell_ch, doctor_cell - from server.core import pg + from server.cli import doctor_cell + from server.core import cell_ch, pg async def topology(_ch: Any) -> Any: return cell_ch.Topology("", 0) diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py index 469ed16ea3..f63f195fda 100644 --- a/tests/unit/test_cell_attach_serving_identity.py +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -10,8 +10,8 @@ import pytest from server.cli import cell_attach, cell_control -from server.cli.cell_ch import Refused from server.core import pg +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FakeConn, _args diff --git a/tests/unit/test_cell_scan.py b/tests/unit/test_cell_scan.py index 56bf01a172..0366a44455 100644 --- a/tests/unit/test_cell_scan.py +++ b/tests/unit/test_cell_scan.py @@ -15,10 +15,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused, Topology -from server.cli.cell_scan import Budget, ScanResult, Table, TableList -from server.core import cell, pg +from server.cli import cell_scan +from server.cli.cell_scan import ScanResult, Table, TableList +from server.core import cell, cell_ch, pg +from server.core.cell_ch import Refused, Topology +from server.core.cell_sync import Budget +from server.insights.store import cell_catalog ONE = Topology("default", 1, (("localhost", 9000),)) TWO = Topology("main", 1, (("ch-0", 9000), ("ch-1", 9000))) @@ -178,7 +180,7 @@ async def test_each_host_answers_for_exactly_one_listed_name() -> None: [{"h": "os-0", "names": ["ch-0"]}, {"h": "os-1", "names": ["ch-1"]}], ) ) - assert await cell_writers.host_names(ch, TWO) == NAMES + assert await cell_catalog.host_names(ch, TWO) == NAMES # Every clusterAllReplicas read waits for every replica and reads its own host locally. reads = [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] assert "skip_unavailable_shards = 0, prefer_localhost_replica = 1" in reads[0] @@ -190,7 +192,7 @@ async def test_a_user_without_read_on_remote_is_named_before_any_cluster_read() ] ch = _granted(FakeCH(), narrow) with pytest.raises(Refused, match="attach's own user lacks READ ON REMOTE"): - await cell_writers.host_names(ch, TWO) + await cell_catalog.host_names(ch, TWO) assert not [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] @@ -208,7 +210,7 @@ async def test_a_host_without_one_configured_name_refuses( rows: list[dict[str, Any]], reason: str ) -> None: with pytest.raises(Refused, match=reason): - await cell_writers.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) + await cell_catalog.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) # -- the table list ---------------------------------------------------------------------- @@ -737,7 +739,7 @@ async def test_two_servers_under_one_host_name_refuse_before_any_read() -> None: same = Topology("main", 1, (("ch-0", 9000), ("ch-0", 9001))) ch = FakeCH() with pytest.raises(Refused, match="lists host ch-0 2 times"): - await cell_writers.host_names(ch, same) + await cell_catalog.host_names(ch, same) listed = await cell_scan.list_tables(ch, None, same, {}) assert [r.reason for r in listed.refusals] == [ "lists host ch-0 2 times (ports 9000, 9001): give each server its own host name" diff --git a/tests/unit/test_cell_scim_precondition.py b/tests/unit/test_cell_scim_precondition.py index 2c0bb0946f..0cf91e4792 100644 --- a/tests/unit/test_cell_scim_precondition.py +++ b/tests/unit/test_cell_scim_precondition.py @@ -11,8 +11,8 @@ from server.auth import authz, revocation, scim, subject_revocation from server.cli import cell_attach, cell_attach_beats -from server.cli.cell_ch import Refused from server.core import cell, ch +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FIELDS, FakeConn, _args, _beat, _value from tests.unit.test_scim_people import People diff --git a/tests/unit/test_cell_scim_precondition_failures.py b/tests/unit/test_cell_scim_precondition_failures.py index 11f46f0f60..ed175cc289 100644 --- a/tests/unit/test_cell_scim_precondition_failures.py +++ b/tests/unit/test_cell_scim_precondition_failures.py @@ -9,7 +9,7 @@ from server.auth import revocation from server.auth.revocation import scim_revoke as actual_scim_revoke -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from tests.unit.test_cell_attach_serving_identity import serving_run as serving_run from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py index 5465767496..353dc1e2cf 100644 --- a/tests/unit/test_cell_update_participants.py +++ b/tests/unit/test_cell_update_participants.py @@ -196,15 +196,15 @@ async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology events = [] topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(ch, top, names, sql, params): if "source:tables" in sql: @@ -235,7 +235,7 @@ async def query(sql, params): return [participant(1), participant(2), participant(1, "drained", version=2)] monkeypatch.setattr(module, "per_host", per_host) - monkeypatch.setattr(module.cell_scan, "_system", sync) + monkeypatch.setattr(module.cell_sync, "_system", sync) result = await module.census(SimpleNamespace(query_all=query)) assert len(result) == 2 and result[str(UUID(int=1))]["state"] == "drained" assert events == (["sync", "read"] if engine.startswith("Replicated") else ["read"]) @@ -251,7 +251,7 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology topology = Topology( "replicas", @@ -260,9 +260,9 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey ) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(*args): if problem == "unreachable": @@ -293,7 +293,7 @@ async def per_host(*args): monkeypatch.setattr(module, "per_host", per_host) monkeypatch.setattr( - module.cell_scan, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) + module.cell_sync, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) ) query = AsyncMock(side_effect=AssertionError("must not read partial census")) with pytest.raises((module.cell_control.Refused, OSError)): diff --git a/tests/unit/test_cell_update_scim_runtime.py b/tests/unit/test_cell_update_scim_runtime.py index ebcfbeb3da..a59337b8ae 100644 --- a/tests/unit/test_cell_update_scim_runtime.py +++ b/tests/unit/test_cell_update_scim_runtime.py @@ -12,9 +12,9 @@ from server.auth import scim from server.cli import cell_control, cell_update_participants, cell_update_probe, cell_update_run -from server.cli.cell_ch import Refused from server.cli.cell_update_pg import State from server.core import cell, pg_profile +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.cell_control_owner import owner from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_writers.py b/tests/unit/test_cell_writers.py index a3e85bca42..e1c84a457c 100644 --- a/tests/unit/test_cell_writers.py +++ b/tests/unit/test_cell_writers.py @@ -11,9 +11,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Topology -from server.cli.cell_writers import Allowed, Grant, holds, parse_show_grants, writer_reason +from server.cli import cell_scan, cell_writers +from server.cli.cell_writers import Allowed, writer_reason +from server.core import cell_ch +from server.core.cell_ch import Topology +from server.insights.store import cell_catalog +from server.insights.store.cell_catalog import Grant, holds, parse_show_grants from tests.unit.test_cell_scan import NAMES, ONE, TWO, FakeCH DBS = ["loom", "loom_telemetry"] @@ -69,7 +72,7 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: "REVOKE SELECT ON loom.secret FROM attach", "REVOKE GRANT OPTION FOR SELECT ON other.* FROM attach", ] - assert cell_writers.own_problems(parse_show_grants(lines), ONE, DBS) == [ + assert cell_catalog.own_problems(parse_show_grants(lines), ONE, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW TABLES ON side.*)", "SELECT ON loom.* (narrowed by REVOKE SELECT ON loom.secret)", ] @@ -90,8 +93,8 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: rows.append( {"user_name": "attach", "access_type": "SHOW", "database": "side", "is_partial_revoke": 1} ) - grants = cell_writers.resolve_grants("attach", rows, []) - assert cell_writers.own_problems(grants, TWO, DBS) == [ + grants = cell_catalog.resolve_grants("attach", rows, []) + assert cell_catalog.own_problems(grants, TWO, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW ON side.*)", "SHOW COLUMNS ON *.* (narrowed by REVOKE SHOW ON side.*)", ] @@ -101,10 +104,10 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: def test_attachs_own_user_needs_every_read_its_checks_make() -> None: full = [Grant("SELECT"), Grant("SHOW"), Grant("SHOW ACCESS"), Grant("SYSTEM")] - assert cell_writers.own_problems(full, ONE, DBS) == [] - assert cell_writers.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] + assert cell_catalog.own_problems(full, ONE, DBS) == [] + assert cell_catalog.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] narrow = [Grant("SELECT", "loom", "memory"), Grant("SELECT", "system"), Grant("SHOW")] - assert cell_writers.own_problems(narrow, Topology("", 0), DBS) == [ + assert cell_catalog.own_problems(narrow, Topology("", 0), DBS) == [ "SHOW USERS ON *.*", "SHOW ROLES ON *.*", "SHOW ROW POLICIES ON *.*", @@ -168,7 +171,7 @@ def test_a_grants_row_as_newer_servers_write_it() -> None: {"user_name": "u", "access_type": "WRITE", "access_object": "URL", "database": None}, {"user_name": "u", "access_type": "INSERT", "database": "lo", "is_wildcard": 1}, ] - assert cell_writers.resolve_grants("u", rows, []) == [ + assert cell_catalog.resolve_grants("u", rows, []) == [ Grant("REMOTE"), Grant("URL"), Grant("INSERT", "lo*"), @@ -190,9 +193,9 @@ def test_attachs_own_sessions_have_only_its_default_roles() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "spare", "granted_role_is_default": 0}, ] # fmt: skip - every = cell_writers.resolve_grants("u", grant_rows, role_rows) + every = cell_catalog.resolve_grants("u", grant_rows, role_rows) assert {g.access for g in every} == {"SHOW", "SELECT", "INSERT"} - session = cell_writers.resolve_grants("u", grant_rows, role_rows, default_only=True) + session = cell_catalog.resolve_grants("u", grant_rows, role_rows, default_only=True) assert {g.access for g in session} == {"SHOW", "SELECT"} @@ -203,7 +206,7 @@ def test_a_role_named_as_a_user_is_not_that_user() -> None: ] # Role `attach` holds role `w`; user `attach` holds neither. role_rows = [{"user_name": None, "role_name": "attach", "granted_role_name": "w"}] - assert cell_writers.resolve_grants("attach", grant_rows, role_rows) == [] + assert cell_catalog.resolve_grants("attach", grant_rows, role_rows) == [] def test_roles_are_followed_and_partial_revokes_ignored() -> None: @@ -224,7 +227,7 @@ def test_roles_are_followed_and_partial_revokes_ignored() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "outer"}, {"user_name": None, "role_name": "outer", "granted_role_name": "inner"}, ] - assert cell_writers.resolve_grants("u", grant_rows, role_rows) == [ + assert cell_catalog.resolve_grants("u", grant_rows, role_rows) == [ Grant("INSERT", "loom"), Grant("SELECT"), Grant("SELECT", "loom", "secret", revoked=True), @@ -470,7 +473,7 @@ def test_a_replica_keeper_lists_outside_the_cluster_is_extra() -> None: "replica_is_active": {"r0": 1}, }, ] - assert cell_writers.extra_replicas(rows) == {"loom.memory": {"dr"}} + assert cell_catalog.extra_replicas(rows) == {"loom.memory": {"dr"}} def _row(user: str, access: str, database: str | None = None) -> dict[str, Any]: @@ -775,7 +778,7 @@ def test_loom_s_own_row_policy_pair_lets_attach_read_every_row() -> None: _policy(POLICY_NAME, select_filter="org = 'a'", apply_to_list=["loom_analyst"]), _policy(OTHERS_POLICY_NAME, apply_to_all=1, apply_to_except=["loom_analyst"]), ] - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] def test_any_row_policy_on_a_system_table_refuses() -> None: @@ -783,7 +786,7 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: _policy("own", database="system", table="grants", apply_to_all=1), _policy("loom_analyst_others", database="system", table="tables", apply_to_all=1), ] - assert cell_writers.row_policy_problems(rows, "attach") == [ + assert cell_catalog.row_policy_problems(rows, "attach") == [ ("row policy on system.grants: own", "a row policy hides rows from the checks"), ( "row policy on system.tables: loom_analyst_others", @@ -839,13 +842,13 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: def test_a_row_policy_hiding_rows_from_attach_refuses( rows: list[dict[str, Any]], reason: str ) -> None: - found = cell_writers.row_policy_problems(rows, "attach") + found = cell_catalog.row_policy_problems(rows, "attach") assert any(reason in why for _, why in found), found async def test_the_row_policies_read_covers_the_system_tables() -> None: ch = _cluster() - await cell_writers.own_refusals(ch, TWO, NAMES) + await cell_catalog.own_refusals(ch, TWO, NAMES) (sql,) = [s for s in ch.sql("query_all") if "system.row_policies" in s] assert "database IN {dbs:Array(String)} OR database = 'system'" in sql @@ -924,14 +927,14 @@ def _looms_policies() -> list[dict[str, Any]]: def test_the_policies_loom_s_analyst_setup_writes_pass() -> None: rows = _looms_policies() assert {r["table"] for r in rows} >= {"memory", "analysis_canary"} - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] # Without every other user's policy, the canary would read no row for attach. alone = [ r for r in rows if not (r["table"] == "analysis_canary" and r["short_name"] == "loom_analyst_others") ] - assert cell_writers.row_policy_problems(alone, "attach") == [ + assert cell_catalog.row_policy_problems(alone, "attach") == [ ( "row policy on loom.analysis_canary", "no permissive policy `USING 1` covers attach: it reads no row", diff --git a/tests/unit/test_scim_review118_regressions.py b/tests/unit/test_scim_review118_regressions.py index 1df99b7173..182c13eaba 100644 --- a/tests/unit/test_scim_review118_regressions.py +++ b/tests/unit/test_scim_review118_regressions.py @@ -7,7 +7,7 @@ import pytest from server.auth import aliases, jit, login, oauth -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run from tests.unit.test_scim_people import people as people From 609f5f70d010bbb6a908ffb53c60882ecd30c884 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 19:30:24 +0000 Subject: [PATCH 177/181] Bound complete cell process census across polls Retain every captured process obligation within one count, metadata and deadline allowance. Share the ordinary participant selection and strict fingerprint codec without changing update policy. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 24 + server/cli/cell_control.py | 4 +- server/cli/cell_update_participants.py | 109 +---- server/cli/cell_update_pg.py | 5 +- server/core/cell.py | 46 ++ server/core/cell_ch.py | 45 +- server/core/ch_bounded.py | 112 +++++ server/insights/domain/cell_observations.py | 294 +++++++++++ server/insights/domain/cell_participants.py | 60 +++ server/insights/store/cell_observations.py | 459 ++++++++++++++++++ .../insights/store/provider_participants.py | 194 +++++++- server/router/app.py | 6 +- server/router/directory.py | 2 +- server/router/fingerprints.py | 54 +-- .../test_cell_fingerprint_reservations.py | 12 +- tests/unit/test_cell_observation_census.py | 295 +++++++++++ .../test_cell_observation_census_stream.py | 245 ++++++++++ tests/unit/test_cell_update_participants.py | 6 +- tests/unit/test_ch_bounded_stream.py | 101 ++++ 19 files changed, 1895 insertions(+), 178 deletions(-) create mode 100644 server/core/ch_bounded.py create mode 100644 server/insights/domain/cell_observations.py create mode 100644 server/insights/domain/cell_participants.py create mode 100644 server/insights/store/cell_observations.py create mode 100644 tests/unit/test_cell_observation_census.py create mode 100644 tests/unit/test_cell_observation_census_stream.py create mode 100644 tests/unit/test_ch_bounded_stream.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index 72e465951c..a0197b53dc 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1620,6 +1620,30 @@ before any cell's processes do, since a prepared first cell asks the state route first cell-state read until it reads `serving`, and only a router of that release answers it from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). +## Bounded process census + +The shared process reader has an explicit bounded mode for complete process evidence. +It reads permanent participants, every stored cell heartbeat and every stored web +instance across the configured replicas, then reads participants again. Replicated +participant tables synchronize before selection; shared tables use sequential reads. +Catalog, membership, privilege or conflicting-version uncertainty refuses the read. +The ordinary update reader keeps its existing query and allowance. + +Each complete read shares an 8 MiB raw allowance across its streams. A caller keeps one +attempt across polls and physical rounds: at most 1024 participant identities, 1024 +observation identities and 8 MiB retained metadata, including certified retirements. +Repeated evidence replaces bounded latest state without evicting captured obligations. +The reader binds the original monotonic deadline and accepts at most 64 configured +replicas. Oversized, truncated or malformed results refuse rather than yield a subset. + +Supported process observations bind the actual participant, certified birth, cell, +accepted fingerprints and root-key fingerprint. Workers join without web instance rows; +traffic-serving first-cell servers also need their router marker. Missing, stale, +future or nonaccepting observations remain unresolved obligations. Native commands +without observations require positive retirement; this component grants no owner +exemption. Its internal result carries unresolved identities and selected evidence. +It performs no attach, exposes no diagnostic route and certifies no physical isolation. + ## Invariants - An org is served by exactly one cell, the one its placement names, and its placement diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index f7f97781c0..9dd6048cf0 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -14,13 +14,11 @@ from server.cli import cell_attach_pg as apg from server.core import pg -from server.core.cell_ch import Refusal +from server.core.cell_ch import TRANSACTION_S, Refusal from server.core.cell_ch import Refused as Refused from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings -# A control transaction has few metadata statements; COMMIT's standby wait is bounded too. -TRANSACTION_S = 60.0 # not a setting: a single control transaction's deadline OPEN_S = 75.0 # not a setting: connection initialization and direct-session checks REQUEST_S = CONTROL_MAX_S + 30.0 # not a setting: directory bound plus probes/connect/close Call = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index b948d47e8e..3e8afbc6af 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -10,8 +10,8 @@ from uuid import UUID from server.cli import cell_control -from server.core import cell, cell_sync, ch_names, instances -from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology +from server.core import cell, instances +from server.core.cell_ch import complete from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -19,8 +19,9 @@ participant_tag, proves_stopped, ) -from server.insights.store import cell_catalog +from server.insights.domain.cell_participants import latest from server.insights.store import provider_participants as participants +from server.insights.store.provider_participants import census class CompleteReads: @@ -36,44 +37,6 @@ async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: ) -def complete(sql: str, *, sequential: int = 0) -> str: - return ( - sql - + " " - + WHOLE_SQL - + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " - + f"select_sequential_consistency={sequential}" - ) - - -def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - out: dict[str, dict[str, Any]] = {} - # The latest winner cannot hide a conflicting older version from another replica. - seen: dict[tuple[str, int], tuple[str, str]] = {} - for row in rows: - try: - identity = str(UUID(str(row["participant_id"]))) - version = row["version"] - state, proof = row["state"], row["host_proof"] - except (KeyError, TypeError, ValueError): - raise cell_control._refused("participant census contains an unreadable row") from None - if ( - type(version) is not int - or version <= 0 - or state not in ("running", "drained", "retired") - or not isinstance(proof, str) - ): - raise cell_control._refused("participant census contains an unreadable row") - previous = out.get(identity) - observed = seen.get((identity, version)) - if observed is not None and observed != (state, proof): - raise cell_control._refused(f"participant {identity} has conflicting equal versions") - seen[(identity, version)] = (state, proof) - if previous is None or int(previous["version"]) < version: - out[identity] = dict(row) - return out - - def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: return { identity: row @@ -82,70 +45,6 @@ def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: } -async def census(ch: Any) -> dict[str, dict[str, Any]]: - """Synchronize after the durable cutoff; every configured replica must answer.""" - topology = await read_topology(ch) - if topology.cluster and not topology.entries: - raise cell_control._refused("participant census cannot resolve the configured cluster") - if problems := topology.refusals(): - raise Refused(problems) - names = await cell_catalog.host_names(ch, topology) if topology.multi else {} - if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: - raise Refused(problems) - database = ch_names.database() - tables = await per_host( - ch, - topology, - names, - ( - "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " - "name='_provider_participants'" - ), - {"db": database}, - ) - if not tables or any(len(rows) != 1 for rows in tables.values()): - raise cell_control._refused("participant table is missing from a configured replica") - engines = {str(rows[0]["engine"]) for rows in tables.values()} - if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): - raise cell_control._refused( - "participant replicas have unreadable or different table engines" - ) - engine = next(iter(engines)) - if engine.startswith("Replicated"): - rows = await per_host( - ch, - topology, - names, - ( - "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " - "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " - "table='_provider_participants'" - ), - {"db": database}, - ) - if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( - row for value in rows.values() for row in value - ): - raise cell_control._refused("participant replica set includes an unverified replica") - budget = cell_sync.Budget(cell_control.TRANSACTION_S) - await cell_sync._system( - ch, - f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " - f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", - topology, - budget, - "participant census", - ) - source = f"{quote(database)}.`_provider_participants`" - params = {} - if topology.multi: - source = f"clusterAllReplicas({{cluster:String}}, {source})" - params["cluster"] = topology.cluster - sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" - rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) - return latest(rows) - - async def stopped(ch: Any, initial: dict[str, Any], current: dict[str, Any] | None) -> bool: if current is not None: try: diff --git a/server/cli/cell_update_pg.py b/server/cli/cell_update_pg.py index ce7cdebcde..fb5f16aa93 100644 --- a/server/cli/cell_update_pg.py +++ b/server/cli/cell_update_pg.py @@ -8,7 +8,6 @@ from server.cli import cell_control from server.core import cell -from server.router import fingerprints @dataclass(frozen=True) @@ -49,8 +48,8 @@ async def read(conn: Any, name: str) -> State: "update needs complete prepared state naming this cell; prepare or restore it first" ) current, pending = cell.json_object(row["current_fp"]), cell.json_object(row["next_fp"]) - if not fingerprints.readable(current) or ( - pending is not None and not fingerprints.readable(pending) + if not cell.readable_fingerprint(current) or ( + pending is not None and not cell.readable_fingerprint(pending) ): raise cell_control._refused("update needs readable local current/next fingerprints") assert current is not None diff --git a/server/core/cell.py b/server/core/cell.py index eb996a5fcc..9de44e23ec 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -240,6 +240,52 @@ def matches(mine: Mapping[str, Any], value: Mapping[str, Any] | None) -> bool: return True +def readable_fingerprint(value: Any) -> bool: + """Known fields have exact types; unknown JSON fields remain reserved and intact.""" + if not isinstance(value, dict) or set(value) != {"v", "fields"}: + return False + if type(value["v"]) is not int or value["v"] != FINGERPRINT_VERSION: + return False + fields = value["fields"] + if not isinstance(fields, dict): + return False + try: + # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + return False + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + try: + if "\0" in item: + return False + item.encode() + except UnicodeError: + return False + elif isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + return False + pending.extend(item.keys()) + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + elif item is not None and type(item) not in (bool, int, float): + return False + for name, default in DEFAULTS.items(): + item = fields.get(name, default) + if type(item) is not type(default): + return False + if isinstance(item, list) and any(not isinstance(part, str) for part in item): + return False + for name in ("configured_orgs", "end_user_orgs"): + for org in fields.get(name, []): + if named_org(org) is None or org_name_refusal(org): + return False + return True + + def digest(fields: Mapping[str, Any]) -> str: """A short digest of a fingerprint for the gate's reason; never compared.""" blob = json.dumps(dict(fields), sort_keys=True, separators=(",", ":")) diff --git a/server/core/cell_ch.py b/server/core/cell_ch.py index 6262a23176..84873b609d 100644 --- a/server/core/cell_ch.py +++ b/server/core/cell_ch.py @@ -13,6 +13,9 @@ from server.core.ch import query_tag from server.settings import settings +# A control transaction includes the standby acknowledgment of COMMIT. +TRANSACTION_S = 60.0 # not a setting: one control transaction's deadline + # system.query_log's log_comment: the scan's, and the writer check's. Attach's own grant # reads carry the tag of the check that runs them; the host names always carry the scan's. TAG = "cell.scan" @@ -130,16 +133,19 @@ def refusals(self) -> list[Refusal]: return out -async def read_topology(ch: Any) -> Topology: +async def read_topology(ch: Any, *, max_replicas: int | None = None) -> Topology: cluster = settings.ch_cluster if not cluster: return Topology("", 0) with query_tag(TAG): - rows = await ch.query_all( - "SELECT host_name, port, shard_num FROM system.clusters WHERE cluster = {c:String} " - + WHOLE_SQL, - {"c": cluster}, - ) + sql = "SELECT host_name, port, shard_num FROM system.clusters WHERE cluster = {c:String} " + params: dict[str, Any] = {"c": cluster} + if max_replicas is not None: + sql += "LIMIT {replica_limit:UInt16} " + params["replica_limit"] = max_replicas + 1 + rows = await ch.query_all(sql + WHOLE_SQL, params) + if max_replicas is not None: + bounded_topology(rows, max_replicas) return Topology( cluster, len({int(r["shard_num"]) for r in rows}), @@ -175,3 +181,30 @@ async def per_host( raise Refused([Refusal(f"host {r['h']}", "answered without a configured name")]) out[name].append(r) return out + + +def complete(sql: str, *, sequential: int = 0) -> str: + return ( + sql + + " " + + WHOLE_SQL + + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " + + f"select_sequential_consistency={sequential}" + ) + + +def bounded_topology(rows: list[dict[str, Any]], limit: int) -> None: + """Reject cap+1 and malformed topology before Topology builds sets or maps.""" + if len(rows) > limit: + raise Refused([Refusal("participant census", "configured replica limit exceeded")]) + for row in rows: + if ( + not isinstance(row.get("host_name"), str) + or not row["host_name"] + or len(row["host_name"].encode()) > 2048 + or type(row.get("port")) is not int + or not 0 < row["port"] <= 65535 + or type(row.get("shard_num")) is not int + or row["shard_num"] <= 0 + ): + raise Refused([Refusal("participant census", "unreadable configured replica")]) diff --git a/server/core/ch_bounded.py b/server/core/ch_bounded.py new file mode 100644 index 0000000000..4799b99b19 --- /dev/null +++ b/server/core/ch_bounded.py @@ -0,0 +1,112 @@ +"""Finite JSONEachRow reads with limits enforced before decoding rows.""" + +from __future__ import annotations + +import asyncio +import json +import math +import time +from typing import Any + +from server.core.ch import own_write_reads, set_query_tag, statement_timeout + + +class BoundedReadError(ValueError): + """The complete answer cannot be established within its read allowance.""" + + +def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise BoundedReadError("Duplicate result field") + result[key] = value + return result + + +def _constant(value: str) -> Any: + raise BoundedReadError("Nonfinite result value") + + +def _float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise BoundedReadError("Nonfinite result value") + return result + + +def decode_json(text: str) -> Any: + """Decode already byte-bounded JSON without duplicate keys or nonfinite numbers.""" + try: + return json.loads( + text, + object_pairs_hook=_object, + parse_constant=_constant, + parse_float=_float, + ) + except (UnicodeError, ValueError, RecursionError) as error: + raise BoundedReadError("Malformed JSON value") from error + + +def _row(line: bytes) -> dict[str, Any]: + try: + row = decode_json(line.decode("utf-8")) + except UnicodeError as error: + raise BoundedReadError("Result row is not UTF-8") from error + if not isinstance(row, dict): + raise BoundedReadError("Result row is not an object") + return row + + +async def read_rows( + sql: str, + params: dict[str, Any], + *, + ch: Any, + deadline: float, + row_bytes: int, + total_bytes: int, + max_rows: int, + tag: str, + max_read_rows: int = 1_000_000, + max_read_bytes: int = 128 * 1024**2, +) -> list[dict[str, Any]]: + """Read a caller's fixed SELECT with its explicit LIMIT and JSONEachRow format.""" + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0 or min(row_bytes, total_bytes, max_rows, max_read_rows, max_read_bytes) <= 0: + raise BoundedReadError("Read allowance exhausted") + limits = { + "max_execution_time": remaining, + "max_memory_usage": 128 * 1024**2, + "max_rows_to_read": min(max_read_rows, 1_000_000), + "max_bytes_to_read": min(max_read_bytes, 128 * 1024**2), + "max_result_rows": max_rows + 1, + "max_result_bytes": total_bytes + 1, + "read_overflow_mode": "throw", + "result_overflow_mode": "throw", + "timeout_overflow_mode": "throw", + "output_format_json_quote_64bit_integers": 0, + } + rows: list[dict[str, Any]] = [] + pending = bytearray() + consumed = 0 + set_query_tag(tag) + async with asyncio.timeout(remaining): + with own_write_reads(), statement_timeout(remaining): + async with ch.raw_stream(sql, params=params, settings=limits) as chunks: + async for chunk in chunks: + consumed += len(chunk) + if consumed > total_bytes: + raise BoundedReadError("Result exceeds its byte bound") + pending.extend(chunk) + while (end := pending.find(b"\n")) >= 0: + if end > row_bytes or len(rows) >= max_rows: + raise BoundedReadError("Result exceeds its row bound") + line = bytes(pending[:end]) + del pending[: end + 1] + rows.append(_row(line)) + if len(pending) > row_bytes: + raise BoundedReadError("Row exceeds its byte bound") + if pending: + raise BoundedReadError("Incomplete result row") + return rows diff --git a/server/insights/domain/cell_observations.py b/server/insights/domain/cell_observations.py new file mode 100644 index 0000000000..0946d607ee --- /dev/null +++ b/server/insights/domain/cell_observations.py @@ -0,0 +1,294 @@ +"""Finite process observations and cumulative obligations for one cell attempt.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any +from uuid import UUID + +from server.core import cell +from server.core.provider_process import certified, has_certificate +from server.insights.domain.cell_participants import bounded_participant, refused + +PARTICIPANTS = 1024 +OBSERVATIONS = 1024 +REPLICAS = 64 +METADATA_BYTES = 8 * 1024**2 +FINGERPRINT_BYTES = 256 * 1024 +FIELD_BYTES = 2048 + + +def text(value: Any, *, empty: bool = False) -> str: + if not isinstance(value, str) or (not empty and not value): + raise refused("unreadable process metadata") + try: + if "\0" in value or len(value.encode()) > FIELD_BYTES: + raise ValueError + except UnicodeError: + raise refused("unreadable process metadata") from None + except ValueError: + raise refused("process metadata exceeds its field bound") from None + return value + + +def identity(value: Any, *, instance: bool = False) -> str: + value = text(value) + try: + parsed = UUID(value) + except ValueError: + raise refused("unreadable process identity") from None + expected = parsed.hex if instance else str(parsed) + if value != expected: + raise refused("noncanonical process identity") + return value + + +def _size(value: Any) -> int: + return len( + json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode() + ) + + +@dataclass(frozen=True) +class Context: + """Already bounded own/directory values, supplied by the normal owning caller.""" + + cell_name: str + is_first: bool + own_pair: tuple[dict[str, Any], ...] + directory_pair: tuple[dict[str, Any], ...] + recorded_keys_fp: str | None + + def validate(self) -> None: + if not isinstance(self.cell_name, str) or not cell.NAME.fullmatch(self.cell_name): + raise refused("unreadable census cell") + if type(self.is_first) is not bool: + raise refused("unreadable census first-cell state") + for pair in (self.own_pair, self.directory_pair): + if not 1 <= len(pair) <= 2 or any(not cell.readable_fingerprint(v) for v in pair): + raise refused("unreadable accepted fingerprint pair") + if _size(pair) > 64 * 1024: + raise refused("accepted fingerprint pair exceeds its bound") + if self.recorded_keys_fp is not None and not cell.SHA256.fullmatch(self.recorded_keys_fp): + raise refused("unreadable recorded root-key fingerprint") + + +def _claim(observed: dict[str, Any]) -> dict[str, Any]: + value = observed["value"] + claim = value.get("attach_observation") + keys = {"protocol", "role", "participant_id", "host_proof", "keys_fp", "accepting", "period_ms"} + if not isinstance(claim, dict) or set(claim) != keys: + raise refused("process reports no supported attach observation") + if type(claim["protocol"]) is not int or claim["protocol"] != 1: + raise refused("process reports an unsupported observation protocol") + if claim["role"] not in ("server", "worker") or type(claim["accepting"]) is not bool: + raise refused("process reports an unreadable observation role or acceptance") + if identity(claim["participant_id"]) != observed["participant_id"]: + raise refused("inner and outer participant identities disagree") + proof = text(claim["host_proof"]) + if not has_certificate(proof): + raise refused("process reports no installed lifecycle certificate") + keys_fp = text(claim["keys_fp"]) + if not cell.SHA256.fullmatch(keys_fp): + raise refused("process reports an unreadable root-key fingerprint") + if type(claim["period_ms"]) is not int or not 1000 <= claim["period_ms"] <= 60000: + raise refused("process reports an unsupported heartbeat period") + if type(value.get("serves")) is not bool or type(value.get("marked")) is not bool: + raise refused("process reports unreadable serving or marker state") + if claim["role"] == "worker" and value["serves"]: + raise refused("worker observation claims to serve traffic") + return claim + + +@dataclass +class Attempt: + """One invocation retains first obligations and latest evidence without eviction.""" + + participants: dict[str, dict[str, Any]] = field(default_factory=dict) + observations: dict[str, dict[str, Any]] = field(default_factory=dict) + _sizes: dict[tuple[str, str], int] = field(default_factory=dict) + retained_bytes: int = 0 + deadline: float | None = None + cell_name: str | None = None + + def _retain(self, kind: str, key: str, value: dict[str, Any]) -> None: + table = self.participants if kind == "participant" else self.observations + limit = PARTICIPANTS if kind == "participant" else OBSERVATIONS + if key not in table and len(table) >= limit: + raise refused(f"cumulative {kind} identity limit exceeded") + size = _size(value) + total = self.retained_bytes - self._sizes.get((kind, key), 0) + size + if total > METADATA_BYTES: + raise refused("cumulative process metadata byte limit exceeded") + table[key] = value + self._sizes[(kind, key)] = size + self.retained_bytes = total + + def capture(self, rows: dict[str, dict[str, Any]]) -> None: + for key, raw in rows.items(): + current = bounded_participant(raw) + if key != current["participant_id"]: + raise refused("participant census key disagrees with its row") + birth = certified(current["host_proof"]) + previous = self.participants.get(key) + if previous: + if previous["birth"] != birth: + raise refused("participant UUID is reused across process births") + latest = previous["latest"] + if current["version"] < latest["version"]: + raise refused("participant version regressed") + if current["version"] == latest["version"] and current != latest: + raise refused("participant has conflicting equal versions across reads") + if previous["closed"] and current["state"] != "retired": + raise refused("a certified retired participant re-enrolled") + closed = current["state"] == "retired" and has_certificate(current["host_proof"]) + first = ( + previous["first"] if previous else {"birth": birth, "version": current["version"]} + ) + self._retain( + "participant", + key, + {"first": first, "latest": current, "birth": birth, "closed": closed}, + ) + + def retain_observation(self, observed: dict[str, Any]) -> None: + key = observed["instance_id"] + previous = self.observations.get(key) + participant_id = observed["participant_id"] + participant = self.participants.get(participant_id) + if participant is None: + raise refused("observation has no participant in the complete census") + # A retired historical row need not acquire an observation capability retroactively. + if not participant["closed"]: + claim = _claim(observed) + if claim["host_proof"] != participant["birth"]: + raise refused("observation and participant process births disagree") + digest = hashlib.sha256( + json.dumps( + {k: v for k, v in observed.items() if k not in {"age_ms", "generation"}}, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + if previous: + if previous["participant_id"] != participant_id: + raise refused("observation identity is reused by another participant") + if observed["generation"] < previous["generation"]: + raise refused("observation generation regressed") + if observed["generation"] == previous["generation"] and digest != previous["digest"]: + raise refused("observation has conflicting equal generations") + for other_key, other in self.observations.items(): + if other_key != key and other["participant_id"] == participant_id: + raise refused("participant reused its observation identity") + # Retain identity/generation evidence, not a history of fingerprint payloads. + self._retain( + "observation", + key, + { + "participant_id": participant_id, + "generation": observed["generation"], + "digest": digest, + }, + ) + + def judge( + self, + context: Context, + current: dict[str, dict[str, Any]], + observed: dict[str, dict[str, Any]], + instances: dict[str, dict[str, Any]], + ) -> tuple[tuple[str, ...], str | None]: + """Return unresolved obligations, never a diagnostic or attach-success flag.""" + context.validate() + pending: set[str] = set() + by_participant: dict[str, dict[str, Any]] = {} + keys: set[str] = set() + for value in observed.values(): + participant = value["participant_id"] + if participant in by_participant: + raise refused("participant has more than one process observation") + by_participant[participant] = value + if self.participants[participant]["closed"]: + continue + claim = _admitted(context, value) + if not claim["accepting"] or not 0 <= value["age_ms"] <= 3 * claim["period_ms"]: + pending.add(participant) + if not _instance_join(value, claim, instances): + pending.add(participant) + keys.add(claim["keys_fp"]) + for key, participant in self.participants.items(): + if not participant["closed"] and ( + key not in current + or key not in by_participant + or not has_certificate(current[key]["host_proof"]) + ): + pending.add(key) + for key, instance in instances.items(): + if not instance["stopped"]: + joined = observed.get(key) + if joined is None: + pending.add("instance:" + key) + elif ( + self.participants[joined["participant_id"]]["closed"] + and instance["age_ms"] <= 180000 + ): + pending.add("instance:" + key) + if len(keys) > 1: + raise refused("processes report different root-key fingerprints") + actual_keys = next(iter(keys)) if keys else None + if ( + actual_keys + and context.recorded_keys_fp is not None + and actual_keys != context.recorded_keys_fp + ): + raise refused("process root-key fingerprint differs from its recorded value") + if actual_keys is None: + pending.add("processes") + return tuple(sorted(pending)), actual_keys + + +def _admitted(context: Context, value: dict[str, Any]) -> dict[str, Any]: + claim = _claim(value) + if value["cell"] != context.cell_name or value["mode"] != "enforced": + raise refused("process observation belongs to another or unenforced cell") + fields = value["value"]["fields"] + for pair in (context.own_pair, context.directory_pair): + if not any(cell.matches(fields, accepted) for accepted in pair): + raise refused("process fingerprint disagrees with an accepted pair") + if value["value"]["marked"] is not True: + raise refused("process observation has no tenancy marker") + if ( + context.is_first + and claim["role"] == "server" + and value["value"]["serves"] + and value["value"].get("router") is not True + ): + raise refused("first-cell traffic process has no accepted router key") + return claim + + +def _instance_join( + value: dict[str, Any], + claim: dict[str, Any], + instances: dict[str, dict[str, Any]], +) -> bool: + instance = instances.get(value["instance_id"]) + if claim["role"] == "worker": + if instance is not None: + raise refused("worker observation also owns a web instance row") + return True + if ( + instance is None + or instance["stopped"] + or not 0 <= instance["age_ms"] <= 3 * claim["period_ms"] + ): + return False + if instance["keys_fp"] != claim["keys_fp"]: + raise refused("instance and observation root-key fingerprints disagree") + if instance["serving"] and not value["value"]["serves"]: + raise refused("nonserving observation owns a traffic instance") + return True diff --git a/server/insights/domain/cell_participants.py b/server/insights/domain/cell_participants.py new file mode 100644 index 0000000000..b11cde77c6 --- /dev/null +++ b/server/insights/domain/cell_participants.py @@ -0,0 +1,60 @@ +"""Pure participant versions and the existing cell-control refusal envelope.""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + +from server.core.cell_ch import Refusal, Refused + + +def refused(reason: str) -> Refused: + return Refused([Refusal("cell control", reason)]) + + +def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + # The latest winner cannot hide a conflicting older version from another replica. + seen: dict[tuple[str, int], tuple[str, str]] = {} + for row in rows: + try: + identity = str(UUID(str(row["participant_id"]))) + version = row["version"] + state, proof = row["state"], row["host_proof"] + except (KeyError, TypeError, ValueError): + raise refused("participant census contains an unreadable row") from None + if ( + type(version) is not int + or version <= 0 + or state not in ("running", "drained", "retired") + or not isinstance(proof, str) + ): + raise refused("participant census contains an unreadable row") + previous = out.get(identity) + observed = seen.get((identity, version)) + if observed is not None and observed != (state, proof): + raise refused(f"participant {identity} has conflicting equal versions") + seen[(identity, version)] = (state, proof) + if previous is None or int(previous["version"]) < version: + out[identity] = dict(row) + return out + + +def bounded_participant(row: dict[str, Any]) -> dict[str, Any]: + """The bounded mode retains the actual UUID, birth, state and integral version.""" + from server.core.provider_process import certified + + try: + identity = str(UUID(row["participant_id"])) + proof = row["host_proof"] + if identity != row["participant_id"] or not isinstance(proof, str): + raise ValueError + if len(proof.encode()) > 2048: + raise ValueError + certified(proof) + if type(row.get("version")) is not int or not 0 < row["version"] < 2**64: + raise ValueError + except (KeyError, TypeError, ValueError, UnicodeError): + raise refused("participant census contains an unreadable birth") from None + # The existing version/state/conflict policy remains the sole selector. + return latest([row])[identity] diff --git a/server/insights/store/cell_observations.py b/server/insights/store/cell_observations.py new file mode 100644 index 0000000000..8d3a8a3004 --- /dev/null +++ b/server/insights/store/cell_observations.py @@ -0,0 +1,459 @@ +"""Complete bounded process census for the explicit later-attach component.""" + +from __future__ import annotations + +import asyncio +import math +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any + +from server.core import cell, ch_names +from server.core.cell_ch import WHOLE, Topology, complete, per_host, quote +from server.core.ch_bounded import BoundedReadError, decode_json, read_rows +from server.insights.domain import cell_observations as domain +from server.insights.domain.cell_participants import refused +from server.insights.store import provider_participants + +TAG = "cell.attach.census" +_TABLES = ("_provider_participants", "cell_heartbeats", "instances") +_COLUMNS = { + "_provider_participants": { + "participant_id": "UUID", + "state": "LowCardinality(String)", + "version": "UInt64", + "host_proof": "String", + }, + "cell_heartbeats": { + "instance_id": "String", + "cell": "String", + "tenancy_mode": "String", + "fingerprint": "String", + "heartbeat_at": "DateTime64(3)", + }, + "instances": { + "instance_id": "String", + "keys_fp": "String", + "serving": "UInt8", + "stopped": "UInt8", + "heartbeat_at": "DateTime64(3)", + }, +} + + +class _Reads: + """One complete census shares one raw allowance across all reads and passes.""" + + def __init__(self, ch: Any, attempt: domain.Attempt, deadline: float) -> None: + remaining = deadline - time.monotonic() + if not math.isfinite(deadline) or not 0 < remaining <= 7200: + raise refused("census has no finite remaining workflow allowance") + if attempt.deadline is not None and deadline != attempt.deadline: + raise refused("census cannot reset its workflow deadline") + attempt.deadline = deadline + self.ch, self.deadline = ch, deadline + self.raw_bytes_used = 0 + + def left(self) -> float: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise refused("census deadline exhausted") + return remaining + + @asynccontextmanager + async def raw_stream( + self, sql: str, params: Any = None, *, settings: Any = None + ) -> AsyncIterator[AsyncIterator[bytes]]: + effective = { + **WHOLE, + "skip_unavailable_shards": 0, + "prefer_localhost_replica": 1, + "default_format": "JSONEachRow", + **(settings or {}), + } + remaining = self.left() + for name in ("max_execution_time", "receive_timeout", "distributed_ddl_task_timeout"): + if name in effective: + effective[name] = min(float(effective[name]), remaining) + async with asyncio.timeout(remaining): + async with self.ch.raw_stream(sql, params=params or {}, settings=effective) as chunks: + + async def limited() -> AsyncIterator[bytes]: + async for chunk in chunks: + if self.raw_bytes_used + len(chunk) > domain.METADATA_BYTES: + raise BoundedReadError("Aggregate census raw byte limit exceeded") + self.raw_bytes_used += len(chunk) + yield chunk + + yield limited() + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + values = params or {} + # A JSONEachRow object needs at least three bytes: {} followed by newline. + max_rows = domain.METADATA_BYTES // 3 + if "replica_limit" in values: + max_rows = domain.REPLICAS + 1 + if "LIMIT 1025 BY h" in sql: + max_rows = (domain.PARTICIPANTS + 1) * domain.REPLICAS + result = await read_rows( + sql, + values, + ch=self, + deadline=self.deadline, + row_bytes=domain.FINGERPRINT_BYTES + 32 * domain.FIELD_BYTES, + total_bytes=domain.METADATA_BYTES - self.raw_bytes_used, + max_rows=max_rows, + tag=TAG, + ) + self.left() + _metadata_fields(result) + if "groupArrayIf(host_name, is_local)" in sql: + _host_names(result) + return result + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any] | None: + rows = await self.query_all(sql, params) + if len(rows) > 1: + raise refused("single-row census query returned more than one row") + return rows[0] if rows else None + + async def raw_bytes(self, sql: str, params: Any = None, *, settings: Any = None) -> bytes: + chunks = [] + async with self.raw_stream(sql, params, settings=settings) as stream: + async for chunk in stream: + chunks.append(chunk) + body = b"".join(chunks) + lines = body.splitlines() + if len(lines) > domain.REPLICAS: + raise refused("replica synchronization status limit exceeded") + for line in lines: + if len(line) > domain.FINGERPRINT_BYTES: + raise refused("replica synchronization status row exceeds its bound") + value = decode_json(line.decode("utf-8")) + if not isinstance(value, dict): + raise refused("unreadable replica synchronization status") + _metadata_fields([value]) + return body + + async def execute(self, sql: str, params: Any = None, *, settings: Any = None) -> None: + remaining = self.left() + bounds = dict(settings or {}) + for name in ("max_execution_time", "receive_timeout"): + bounds[name] = min(float(bounds.get(name, remaining)), remaining) + async with asyncio.timeout(remaining): + await self.ch.execute(sql, params, settings=bounds) + + +def _metadata_fields(rows: list[dict[str, Any]]) -> None: + # Fingerprint has its own larger bound and strict nested decoder before retention. + for row in rows: + for key, value in row.items(): + domain.text(key) + if key != "fingerprint": + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + domain.text(item, empty=True) + elif isinstance(item, list): + pending.extend(item) + elif isinstance(item, dict): + for name, child in item.items(): + domain.text(name) + pending.append(child) + + +def _host_names(rows: list[dict[str, Any]]) -> None: + if len(rows) > domain.REPLICAS: + raise refused("answering replica limit exceeded") + for row in rows: + domain.text(row.get("h")) + names = row.get("names") + if not isinstance(names, list) or len(names) != 1: + raise refused("answering replica has no unique configured name") + domain.text(names[0]) + + +async def _catalog(ch: _Reads, topology: Topology, names: dict[str, str]) -> None: + params = {"db": ch_names.database(), "tables": list(_TABLES)} + tables = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, name, engine, engine_full FROM {source:tables} " + "WHERE database={db:String} AND name IN {tables:Array(String)}", + params, + ) + columns = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, table, name, type FROM {source:columns} " + "WHERE database={db:String} AND table IN {tables:Array(String)}", + params, + ) + reference = None + for host, found in tables.items(): + if len(found) != len(_TABLES) or len({r.get("name") for r in found}) != len(_TABLES): + raise refused("census table catalog is missing or duplicated") + kinds = {domain.text(r["name"]): domain.text(r["engine"]) for r in found} + if kinds.get("cell_heartbeats") != "KeeperMap" or kinds.get("instances") != "KeeperMap": + raise refused("observation or instance table is not KeeperMap") + typed = tuple( + sorted( + (domain.text(r["table"]), domain.text(r["name"]), domain.text(r["type"])) + for r in columns.get(host, []) + ) + ) + if len({(t, n) for t, n, _ in typed}) != len(typed): + raise refused("census column catalog is duplicated") + for table, expected in _COLUMNS.items(): + actual = {n: kind for t, n, kind in typed if t == table} + if any(actual.get(name) != kind for name, kind in expected.items()): + raise refused("census table has an unsupported column type") + keeper = tuple( + sorted( + (r["name"], domain.text(r["engine_full"])) + for r in found + if r["name"] != "_provider_participants" + ) + ) + view = (tuple(sorted(kinds.items())), typed, keeper) + if reference is not None and reference != view: + raise refused("census replicas disagree on their table catalog") + reference = view + + +def _source(table: str, topology: Topology) -> tuple[str, dict[str, Any]]: + source = f"{quote(ch_names.database())}.`{table}`" + if topology.multi: + return f"clusterAllReplicas({{cluster:String}}, {source})", {"cluster": topology.cluster} + return source, {} + + +async def _observations(ch: _Reads, topology: Topology) -> list[dict[str, Any]]: + source, params = _source("cell_heartbeats", topology) + sql = ( + "SELECT hostName() AS h, " + "if(length(census_observation.instance_id)<=2048, " + "census_observation.instance_id, '') AS instance_id, " + "if(length(census_observation.cell)<=2048, census_observation.cell, '') AS " + "cell, length(census_observation.cell) AS cell_bytes, " + "if(length(census_observation.tenancy_mode)<=2048, " + "census_observation.tenancy_mode, '') AS tenancy_mode, " + "length(census_observation.tenancy_mode) AS tenancy_mode_bytes, " + "if(length(census_observation.fingerprint)<=262144, " + "census_observation.fingerprint, '') AS fingerprint, " + "length(census_observation.fingerprint) AS fingerprint_bytes, " + "toUnixTimestamp64Milli(census_observation.heartbeat_at) AS generation, " + "dateDiff('millisecond', census_observation.heartbeat_at, now64(3)) AS age_ms " + f"FROM {source} AS census_observation LIMIT 1025 BY h" + ) + return await ch.query_all(complete(sql, sequential=1), params) + + +async def _instances(ch: _Reads, topology: Topology) -> list[dict[str, Any]]: + source, params = _source("instances", topology) + sql = ( + "SELECT hostName() AS h, " + "if(length(census_instance.instance_id)<=2048, census_instance.instance_id, " + "'') AS instance_id, " + "if(length(census_instance.keys_fp)<=2048, census_instance.keys_fp, '') AS keys_fp, " + "length(census_instance.keys_fp) AS keys_fp_bytes, serving, stopped, " + "toUnixTimestamp64Milli(census_instance.heartbeat_at) AS generation, " + "dateDiff('millisecond', census_instance.heartbeat_at, now64(3)) AS age_ms " + f"FROM {source} AS census_instance LIMIT 1025 BY h" + ) + return await ch.query_all(complete(sql, sequential=1), params) + + +def _projected_field(row: dict[str, Any], name: str) -> None: + value = domain.text(row.get(name), empty=True) + size = row.get(name + "_bytes") + if type(size) is not int or not 0 <= size <= domain.FIELD_BYTES or len(value.encode()) != size: + raise refused("process field exceeds or disagrees with its projection bound") + + +def _replicas(rows: list[dict[str, Any]], topology: Topology, names: dict[str, str]) -> None: + counts: dict[str, int] = {} + identities: set[str] = set() + for row in rows: + host = domain.text(row.get("h")) + if topology.multi and host not in names: + raise refused("process row came from an unconfigured replica") + if host not in counts and len(counts) >= (len(names) if topology.multi else 1): + raise refused("process row came from an extra replica") + count = counts.get(host, 0) + 1 + if count > 1024: + raise refused("process replica row limit exceeded") + counts[host] = count + identity = domain.identity(row.get("instance_id"), instance=True) + if identity not in identities and len(identities) >= 1024: + raise refused("logical observation identity limit exceeded") + identities.add(identity) + + +def _collapse_observations( + rows: list[dict[str, Any]], attempt: domain.Attempt +) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + replicas: set[tuple[str, str]] = set() + for row in rows: + _projected_field(row, "cell") + _projected_field(row, "tenancy_mode") + value = observation(row) + key = value["instance_id"] + replica = (row["h"], key) + if replica in replicas: + raise refused("one replica returned a duplicated observation") + replicas.add(replica) + previous = out.get(key) + if previous: + stable = {k: v for k, v in value.items() if k not in {"age_ms", "generation"}} + old = {k: v for k, v in previous.items() if k not in {"age_ms", "generation"}} + if stable != old: + raise refused("replicas disagree on the process observation") + # Every replica must be fresh; keep the worst observed age for the judge. + value["age_ms"] = ( + -1 + if min(value["age_ms"], previous["age_ms"]) < 0 + else max(value["age_ms"], previous["age_ms"]) + ) + value["generation"] = max(value["generation"], previous["generation"]) + attempt.retain_observation(value) + out[key] = value + return out + + +def _collapse_instances(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + replicas: set[tuple[str, str]] = set() + for row in rows: + key = domain.identity(row.get("instance_id"), instance=True) + _projected_field(row, "keys_fp") + value = { + "keys_fp": domain.text(row.get("keys_fp"), empty=True), + **{name: row.get(name) for name in ("serving", "stopped", "age_ms", "generation")}, + } + if ( + any( + type(value[name]) is not int + for name in ("serving", "stopped", "age_ms", "generation") + ) + or value["serving"] not in (0, 1) + or value["stopped"] not in (0, 1) + or value["generation"] < 0 + ): + raise refused("unreadable web instance metadata") + replica = (row["h"], key) + if replica in replicas: + raise refused("one replica returned a duplicated web instance") + replicas.add(replica) + previous = out.get(key) + if previous: + if any(previous[name] != value[name] for name in ("keys_fp", "serving", "stopped")): + raise refused("replicas disagree on a web instance") + value["age_ms"] = ( + -1 + if min(value["age_ms"], previous["age_ms"]) < 0 + else max(value["age_ms"], previous["age_ms"]) + ) + value["generation"] = max(value["generation"], previous["generation"]) + out[key] = value + return out + + +@dataclass(frozen=True) +class Snapshot: + """Complete selected evidence with explicit unresolved obligations; no activation.""" + + participants: dict[str, dict[str, Any]] + observations: dict[str, dict[str, Any]] + pending: tuple[str, ...] + keys_fp: str | None + raw_bytes: int + retained_bytes: int + + +async def read_cell( + ch: Any, attempt: domain.Attempt, context: domain.Context, *, deadline: float +) -> Snapshot: + """Read participants, all observations/instances, then participants again.""" + context.validate() + if attempt.cell_name is not None and attempt.cell_name != context.cell_name: + raise refused("census attempt cannot change its cell") + attempt.cell_name = context.cell_name + reader = _Reads(ch, attempt, deadline) + first = await provider_participants.census(reader, bounded=True) + attempt.capture(first.rows) + await _catalog(reader, first.topology, first.names) + raw_observations = await _observations(reader, first.topology) + _replicas(raw_observations, first.topology, first.names) + raw_instances = await _instances(reader, first.topology) + _replicas(raw_instances, first.topology, first.names) + second = await provider_participants.census(reader, bounded=True) + if (first.topology, first.names, first.engine) != ( + second.topology, + second.names, + second.engine, + ): + raise refused("participant topology changed during the observation join") + attempt.capture(second.rows) + observed = _collapse_observations(raw_observations, attempt) + instances = _collapse_instances(raw_instances) + pending, keys_fp = attempt.judge(context, second.rows, observed, instances) + reader.left() + return Snapshot( + second.rows, observed, pending, keys_fp, reader.raw_bytes_used, attempt.retained_bytes + ) + + +def _finite_fields(value: dict[str, Any]) -> None: + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + domain.text(item, empty=True) + elif isinstance(item, dict): + for key, child in item.items(): + domain.text(key) + pending.append(child) + elif isinstance(item, list): + pending.extend(item) + + +def observation(row: dict[str, Any]) -> dict[str, Any]: + """Decode only the bounded projection; CH evaluates age at the actual read.""" + instance = domain.identity(row.get("instance_id"), instance=True) + raw = row.get("fingerprint") + if not isinstance(raw, str) or type(row.get("fingerprint_bytes")) is not int: + raise refused("unreadable observation fingerprint") + if ( + row["fingerprint_bytes"] > domain.FINGERPRINT_BYTES + or len(raw.encode()) != row["fingerprint_bytes"] + ): + raise refused("observation fingerprint exceeds its bound") + value = decode_json(raw) + if not isinstance(value, dict): + raise refused("unreadable observation fingerprint") + fp = {key: value[key] for key in ("v", "fields") if key in value} + if not cell.readable_fingerprint(fp): + raise refused("unreadable observation fingerprint") + # Fingerprint fields use the strict codec and their aggregate byte allowance. + _finite_fields({key: item for key, item in value.items() if key != "fields"}) + participant = domain.identity(value.get("participant_id")) + if type(row.get("generation")) is not int or row["generation"] < 0: + raise refused("unreadable observation generation") + if type(row.get("age_ms")) is not int: + raise refused("unreadable observation time") + return { + "instance_id": instance, + "participant_id": participant, + "cell": domain.text(row.get("cell"), empty=True), + "mode": domain.text(row.get("tenancy_mode"), empty=True), + "generation": row["generation"], + "age_ms": row["age_ms"], + "value": value, + } diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index e03c5b22bf..34c907b0ba 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -1,9 +1,21 @@ """Full-row process registrations, heartbeats and explicit transition acknowledgements.""" +from dataclasses import dataclass from time import time_ns -from typing import Any +from typing import Any, Literal, overload from uuid import UUID +from server.core import cell_sync, ch_names +from server.core.cell_ch import ( + TRANSACTION_S, + WHOLE_SQL, + Refused, + Topology, + complete, + per_host, + quote, + read_topology, +) from server.core.ch import sequential_reads from server.core.ch_errors import error_code from server.core.provider_process import ( @@ -14,6 +26,8 @@ ) from server.core.repo_base import _CHClient, _default_client from server.core.retirement import RetirementProof, get_gate +from server.insights.domain.cell_participants import bounded_participant, latest, refused +from server.insights.store import cell_catalog from server.settings import settings TABLE = "loom._provider_participants" @@ -226,3 +240,181 @@ def participant_sync(ch: Any, identity: UUID | str) -> dict[str, Any] | None: "version": int(version), "expired": bool(expired), } + + +@dataclass(frozen=True) +class ParticipantCensus: + rows: dict[str, dict[str, Any]] + topology: Topology + names: dict[str, str] + engine: str + + +@overload +async def census(ch: Any, *, bounded: Literal[False] = False) -> dict[str, dict[str, Any]]: ... + + +@overload +async def census(ch: Any, *, bounded: Literal[True]) -> ParticipantCensus: ... + + +async def census( + ch: Any, *, bounded: bool = False +) -> dict[str, dict[str, Any]] | ParticipantCensus: + """Synchronize after the durable cutoff; every configured replica must answer.""" + topology = await read_topology(ch, max_replicas=64) if bounded else await read_topology(ch) + if topology.cluster and not topology.entries: + raise refused("participant census cannot resolve the configured cluster") + if problems := topology.refusals(): + raise Refused(problems) + names = await cell_catalog.host_names(ch, topology) if topology.multi else {} + if bounded and topology.cluster and not topology.multi: + names = await bounded_local_replica(ch, topology) + if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: + raise Refused(problems) + database = ch_names.database() + tables = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " + "name='_provider_participants'" + ), + {"db": database}, + ) + if not tables or any(len(rows) != 1 for rows in tables.values()): + raise refused("participant table is missing from a configured replica") + engines = {str(rows[0]["engine"]) for rows in tables.values()} + if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): + raise refused("participant replicas have unreadable or different table engines") + engine = next(iter(engines)) + if bounded and engine not in ( + "ReplacingMergeTree", + "ReplicatedReplacingMergeTree", + "SharedReplacingMergeTree", + ): + raise refused("participant table has an unsupported replacement policy") + if engine.startswith("Replicated"): + rows = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " + "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " + "table='_provider_participants'" + ), + {"db": database}, + ) + if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( + row for value in rows.values() for row in value + ): + raise refused("participant replica set includes an unverified replica") + if bounded: + bounded_replica_paths(rows) + budget = cell_sync.Budget(TRANSACTION_S) + await cell_sync._system( + ch, + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " + f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", + topology, + budget, + "participant census", + ) + source = f"{quote(database)}.`_provider_participants`" + params = {} + if topology.multi: + source = f"clusterAllReplicas({{cluster:String}}, {source})" + params["cluster"] = topology.cluster + sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" + if bounded: + sql = ( + f"SELECT hostName() AS h, participant_id, state, version, " + "if(length(census_participant.host_proof) <= 2048, " + "census_participant.host_proof, '') AS host_proof, " + f"length(census_participant.host_proof) AS proof_bytes " + f"FROM {source} AS census_participant LIMIT 1025 BY h" + ) + rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) + if bounded: + rows = bounded_participants(rows, topology, names) + return ParticipantCensus(latest(rows), topology, names, engine) + return latest(rows) + + +def bounded_participants( + rows: list[dict[str, Any]], topology: Any, names: Any +) -> list[dict[str, Any]]: + """Validate replica duplication and cap+1 before the shared latest selector.""" + counts: dict[str, int] = {} + logical: set[str] = set() + out = [] + for row in rows: + host = row.get("h") + if not isinstance(host, str) or not host or len(host.encode()) > 2048: + raise refused("participant row has an unreadable replica") + if topology.multi and host not in names: + raise refused("participant row came from an unconfigured replica") + if host not in counts and len(counts) >= (len(names) if topology.multi else 1): + raise refused("participant row came from an extra replica") + count = counts.get(host, 0) + 1 + if count > 1024: + raise refused("participant replica row limit exceeded") + counts[host] = count + if type(row.get("proof_bytes")) is not int or not 0 <= row["proof_bytes"] <= 2048: + raise refused("participant proof exceeds its field bound") + if ( + not isinstance(row.get("host_proof"), str) + or len(row["host_proof"].encode()) != row["proof_bytes"] + ): + raise refused("participant proof length disagrees with its projection") + value = bounded_participant( + {key: row[key] for key in ("participant_id", "state", "version", "host_proof")} + ) + identity = value["participant_id"] + if identity not in logical and len(logical) >= 1024: + raise refused("logical participant limit exceeded") + logical.add(identity) + out.append(value) + return out + + +def bounded_replica_paths(rows: dict[str, list[dict[str, Any]]]) -> None: + """All replicas synchronize one table identity, not unrelated same-named tables.""" + paths: set[str] = set() + names: set[str] = set() + for values in rows.values(): + row = values[0] + path, name = row.get("zookeeper_path"), row.get("replica_name") + if ( + not isinstance(path, str) + or not path + or len(path.encode()) > 2048 + or not isinstance(name, str) + or not name + or len(name.encode()) > 2048 + or name in names + ): + raise refused("participant replica identity is unreadable or duplicated") + paths.add(path) + names.add(name) + if len(paths) != 1: + raise refused("participant replicas do not share one table identity") + for values in rows.values(): + listed = values[0].get("replica_is_active") + if ( + not isinstance(listed, dict) + or set(listed) != names + or any(type(v) is not int or v != 1 for v in listed.values()) + ): + raise refused("participant replica membership is missing, inactive or unreadable") + + +async def bounded_local_replica(ch: Any, topology: Topology) -> dict[str, str]: + """A configured one-host cluster must describe the server read locally.""" + names = await cell_catalog.host_names(ch, topology) + local = await ch.query_one("SELECT hostName() AS h " + WHOLE_SQL) + if local is None or local.get("h") not in names: + raise refused("local participant reader is outside the configured one-host cluster") + return names diff --git a/server/router/app.py b/server/router/app.py index 87cc8d775e..4909335343 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -67,7 +67,7 @@ from server.core import cell as cell_state from server.core.cell_probe import PATH as PROBE_PATH -from server.router import directory, fingerprints, routing +from server.router import directory, routing from server.router.limits import ( ATTACH_BATCH, ATTACH_BEGIN_MAX_BYTES, @@ -831,7 +831,9 @@ async def configuration(self, request: Request) -> JSONResponse: if set(body) != {"action", "target_fp", "keys_fp", "nonces"}: return _refused(400, "configuration requires action, target_fp, keys_fp and nonces") action, target, keys_fp = body["action"], body["target_fp"], body["keys_fp"] - if action not in ("config", "finish", "revert") or not fingerprints.readable(target): + if action not in ("config", "finish", "revert") or not cell_state.readable_fingerprint( + target + ): return _refused(400, "configuration requires a supported action and readable target_fp") if _text(body, "keys_fp", needed=True) is None: return _refused(400, f"keys_fp is text of 1 to {_TEXT_MAX} characters, no NUL") diff --git a/server/router/directory.py b/server/router/directory.py index 0edc5d298a..5fdbc6ad35 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1150,7 +1150,7 @@ async def read(conn: Any, found: Attaching) -> list[str]: accepted = [ value for value in (found.row.current_fp, found.row.next_fp) if value is not None ] - if not accepted or any(not fingerprints.readable(value) for value in accepted): + if not accepted or any(not cell_state.readable_fingerprint(value) for value in accepted): raise OperatorRefused( "configured placement proof needs readable directory reservations" ) diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py index 347ded4a48..2815f4378f 100644 --- a/server/router/fingerprints.py +++ b/server/router/fingerprints.py @@ -10,55 +10,9 @@ from server.router.limits import MAX_BODY_BYTES -def readable(value: Any) -> bool: - """Known fields have exact types; unknown JSON fields remain reserved and intact.""" - if not isinstance(value, dict) or set(value) != {"v", "fields"}: - return False - if type(value["v"]) is not int or value["v"] != cell.FINGERPRINT_VERSION: - return False - fields = value["fields"] - if not isinstance(fields, dict): - return False - try: - # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. - json.dumps(value, allow_nan=False) - except (TypeError, ValueError, RecursionError): - return False - pending = [value] - while pending: - item = pending.pop() - if isinstance(item, str): - try: - if "\0" in item: - return False - item.encode() - except UnicodeError: - return False - elif isinstance(item, dict): - if any(not isinstance(key, str) for key in item): - return False - pending.extend(item.keys()) - pending.extend(item.values()) - elif isinstance(item, list): - pending.extend(item) - elif item is not None and type(item) not in (bool, int, float): - return False - for name, default in cell.DEFAULTS.items(): - item = fields.get(name, default) - if type(item) is not type(default): - return False - if isinstance(item, list) and any(not isinstance(part, str) for part in item): - return False - for name in ("configured_orgs", "end_user_orgs"): - for org in fields.get(name, []): - if cell.named_org(org) is None or cell.org_name_refusal(org): - return False - return True - - def registration_refusal(value: Any, registered: Mapping[str, Any], name: str = "candidate") -> str: """A fingerprint accepts only the registered stores and host.""" - if not readable(value): + if not cell.readable_fingerprint(value): return f"{name}: unreadable fingerprint; change fingerprints with `loom cell update`" expected = { "deployment_id": registered["deployment_id"], @@ -121,7 +75,7 @@ def reservation_refusal(accepted: Sequence[dict[str, Any]], other: Sequence[dict Registration identity, placement winners and transaction locks are the caller's checks. Empty `other` means registration has not reserved a fingerprint yet. """ - if not accepted or any(not readable(value) for value in (*accepted, *other)): + if not accepted or any(not cell.readable_fingerprint(value) for value in (*accepted, *other)): return "unreadable fingerprint reservation" for a in accepted: for b in other: @@ -137,8 +91,8 @@ def configuration_pair( action: str, ) -> tuple[dict[str, Any], dict[str, Any] | None]: """Select the reserved pair without guessing the direction of an interrupted update.""" - if any(not readable(value) for value in (current, target)) or ( - pending is not None and not readable(pending) + if any(not cell.readable_fingerprint(value) for value in (current, target)) or ( + pending is not None and not cell.readable_fingerprint(pending) ): raise ValueError("configuration update needs readable fingerprints") assert current is not None diff --git a/tests/unit/test_cell_fingerprint_reservations.py b/tests/unit/test_cell_fingerprint_reservations.py index 461eadd657..bcdcec3e15 100644 --- a/tests/unit/test_cell_fingerprint_reservations.py +++ b/tests/unit/test_cell_fingerprint_reservations.py @@ -130,7 +130,7 @@ def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: import httpx - from server.router.fingerprints import placement_refusal, readable + from server.router.fingerprints import placement_refusal from server.router.limits import MAX_BODY_BYTES deployment = "11111111-1111-4111-8111-111111111111" @@ -139,7 +139,7 @@ def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: room = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES value["fields"]["oidc_client_id"] = "x" * room org = "\x01" * cell.ORG_MAX_BYTES - assert cell.named_org(org) == org and readable(value) + assert cell.named_org(org) == org and cell.readable_fingerprint(value) live["org"] = org operator = {"org": org, "deployment_id": deployment, "fingerprint": value} assert ( @@ -162,7 +162,7 @@ def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deploymen import httpx - from server.router.fingerprints import placement_refusal, readable + from server.router.fingerprints import placement_refusal from server.router.limits import MAX_BODY_BYTES value = _value(deployment_id=deployment, oidc_client_id="") @@ -173,7 +173,11 @@ def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deploymen room = MAX_BODY_BYTES - max(len(json.dumps(body)) for body in bodies) - 6 * cell.ORG_MAX_BYTES value["fields"]["oidc_client_id"] = "x" * room org = "\x01" * cell.ORG_MAX_BYTES - assert cell.named_org(org) == org and readable(value) and cell.NAME.fullmatch(cell_name) + assert ( + cell.named_org(org) == org + and cell.readable_fingerprint(value) + and cell.NAME.fullmatch(cell_name) + ) for body in bodies: body["org"] = org assert ( diff --git a/tests/unit/test_cell_observation_census.py b/tests/unit/test_cell_observation_census.py new file mode 100644 index 0000000000..d364a807ef --- /dev/null +++ b/tests/unit/test_cell_observation_census.py @@ -0,0 +1,295 @@ +"""A bounded attempt retains every process obligation across complete reads.""" + +from __future__ import annotations + +import copy +import json +from uuid import UUID + +import pytest + +from server.core import cell +from server.core.cell_ch import Refused +from server.insights.domain import cell_observations as domain +from server.insights.store import cell_observations as store + +KEYS = "a" * 64 +PROOF = "host|boot|container|namespace|123|456|||cell-retirement-v1" + + +def participant(n=1, *, state="running", version=1, proof=PROOF): + return { + "participant_id": str(UUID(int=n)), + "state": state, + "version": version, + "host_proof": proof, + } + + +def context(*, first=False): + fp = cell.as_value({"deployment_id": "dep"}) + return domain.Context("later", first, (fp,), (fp,), KEYS) + + +def observation(n=1, *, role="server", age=0, generation=1): + value = { + **cell.as_value({"deployment_id": "dep"}), + "participant_id": str(UUID(int=n)), + "marked": True, + "serves": role == "server", + "router": False, + "attach_observation": { + "protocol": 1, + "role": role, + "participant_id": str(UUID(int=n)), + "host_proof": PROOF, + "keys_fp": KEYS, + "accepting": True, + "period_ms": 1000, + }, + } + raw = json.dumps(value) + return store.observation( + { + "instance_id": UUID(int=n).hex, + "cell": "later", + "tenancy_mode": "enforced", + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "age_ms": age, + "generation": generation, + } + ) + + +def instance(n=1, **changes): + return {UUID(int=n).hex: {"keys_fp": KEYS, "serving": 1, "stopped": 0, "age_ms": 0, **changes}} + + +def test_server_and_worker_join_the_same_complete_attempt(): + attempt = domain.Attempt() + rows = {p["participant_id"]: p for p in (participant(), participant(2))} + attempt.capture(rows) + values = {v["instance_id"]: v for v in (observation(), observation(2, role="worker"))} + for value in values.values(): + attempt.retain_observation(value) + assert attempt.judge(context(), rows, values, instance()) == ((), KEYS) + assert len(attempt.participants) == len(attempt.observations) == 2 + + +@pytest.mark.parametrize( + "fault", + ["stale", "future", "not-accepting", "missing", "disappeared", "stopped", "no-instance"], +) +def test_unknown_or_nonaccepting_processes_remain_obligations(fault): + attempt = domain.Attempt() + row, value = participant(), observation() + rows = {row["participant_id"]: row} + attempt.capture(rows) + if fault == "stale": + value["age_ms"] = 3001 + elif fault == "future": + value["age_ms"] = -1 + elif fault == "not-accepting": + value["value"]["attach_observation"]["accepting"] = False + values = {} if fault == "missing" else {value["instance_id"]: value} + for observed in values.values(): + attempt.retain_observation(observed) + current = {} if fault == "disappeared" else rows + instances = {} if fault == "no-instance" else instance(stopped=int(fault == "stopped")) + pending, _ = attempt.judge(context(), current, values, instances) + assert row["participant_id"] in pending + assert row["participant_id"] in attempt.participants + + +@pytest.mark.parametrize( + "fault", + [ + "protocol", + "bool-period", + "long-period", + "birth", + "uuid", + "worker-instance", + "cell", + "marker", + "fingerprint", + "keys", + "router", + ], +) +def test_inconsistent_observations_refuse(fault): + attempt = domain.Attempt() + row, value = ( + participant(), + observation(role="worker" if fault == "worker-instance" else "server"), + ) + rows = {row["participant_id"]: row} + attempt.capture(rows) + claim = value["value"]["attach_observation"] + if fault == "protocol": + claim["protocol"] = 2 + elif fault == "bool-period": + claim["period_ms"] = True + elif fault == "long-period": + claim["period_ms"] = 60001 + elif fault == "birth": + claim["host_proof"] = PROOF.replace("|123|", "|124|") + elif fault == "uuid": + claim["participant_id"] = str(UUID(int=2)) + elif fault == "cell": + value["cell"] = "other" + elif fault == "marker": + value["value"]["marked"] = False + elif fault == "fingerprint": + value["value"]["fields"]["deployment_id"] = "other" + elif fault == "keys": + claim["keys_fp"] = "b" * 64 + with pytest.raises(Refused): + attempt.retain_observation(value) + attempt.judge( + context(first=fault == "router"), rows, {value["instance_id"]: value}, instance() + ) + + +def test_retirement_closes_only_the_same_captured_birth_and_keeps_its_budget(): + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + closed = participant(state="retired", version=2) + attempt.capture({closed["participant_id"]: closed}) + before = copy.deepcopy(attempt.participants) + assert attempt.judge(context(), {}, {}, {}) == (("processes",), None) + assert attempt.participants == before and len(before) == 1 + with pytest.raises(Refused, match="birth"): + bad = participant(state="retired", version=3, proof=PROOF.replace("|123|", "|124|")) + attempt.capture({bad["participant_id"]: bad}) + assert attempt.participants == before + + +def test_participant_limit_is_cumulative_including_closed_history(): + attempt = domain.Attempt() + for n in range(1, 1025): + row = participant(n, state="retired") + attempt.capture({row["participant_id"]: row}) + size = attempt.retained_bytes + attempt.capture({}) + for row in list(attempt.participants.values()): + current = row["latest"] + attempt.capture({current["participant_id"]: current}) + assert len(attempt.participants) == 1024 and attempt.retained_bytes == size + row = participant(1025) + with pytest.raises(Refused, match="identity limit"): + attempt.capture({row["participant_id"]: row}) + assert len(attempt.participants) == 1024 and attempt.retained_bytes == size + + +def test_observation_limit_is_cumulative_without_evicting_closed_rows(): + attempt = domain.Attempt() + for n in range(1, 1025): + row = participant(n, state="retired") + attempt.capture({row["participant_id"]: row}) + attempt.retain_observation(observation(n)) + before = copy.deepcopy(attempt.observations) + # A different observation identity for an already closed participant is still new history. + value = observation(1) + value["instance_id"] = UUID(int=1025).hex + with pytest.raises(Refused): + attempt.retain_observation(value) + assert len(attempt.observations) == 1024 and attempt.observations == before + + +def test_retained_growth_is_checked_before_replacement(monkeypatch): + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + before = copy.deepcopy(attempt.participants) + monkeypatch.setattr(domain, "METADATA_BYTES", attempt.retained_bytes) + row = participant(version=100000) + with pytest.raises(Refused, match="byte limit"): + attempt.capture({row["participant_id"]: row}) + assert attempt.participants == before + + +def test_repeated_observations_retain_only_latest_evidence_and_check_generation(): + attempt = domain.Attempt() + row, value = participant(), observation() + attempt.capture({row["participant_id"]: row}) + attempt.retain_observation(value) + size = attempt.retained_bytes + attempt.retain_observation(copy.deepcopy(value)) + assert attempt.retained_bytes == size and len(attempt.observations) == 1 + changed = copy.deepcopy(value) + changed["value"]["marked"] = False + with pytest.raises(Refused, match="equal generations"): + attempt.retain_observation(changed) + changed = copy.deepcopy(value) + changed["generation"] = 0 + with pytest.raises(Refused, match="regressed"): + attempt.retain_observation(changed) + assert attempt.retained_bytes == size + + +def test_a_future_replica_cannot_be_hidden_by_a_fresh_replica(): + value = observation() + raw = json.dumps(value["value"]) + rows = [ + { + "h": host, + "instance_id": value["instance_id"], + "cell": "later", + "cell_bytes": 5, + "tenancy_mode": "enforced", + "tenancy_mode_bytes": 8, + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "generation": 1, + "age_ms": age, + } + for host, age in (("a", 0), ("b", -1)) + ] + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + merged = store._collapse_observations(rows, attempt) + assert merged[value["instance_id"]]["age_ms"] == -1 + + +@pytest.mark.parametrize("fault", ["replica", "logical", "unknown-host"]) +def test_replica_and_logical_cap_plus_one_refuse_before_latest_selection(fault): + from server.core.cell_ch import Topology + from server.insights.store.provider_participants import bounded_participants + + count = 1 if fault == "unknown-host" else 1025 + rows = [ + { + **participant(n), + "proof_bytes": len(PROOF.encode()), + "h": "outside" + if fault == "unknown-host" + else "a" + if fault == "replica" + else ("a" if n % 2 else "b"), + } + for n in range(1, count + 1) + ] + with pytest.raises(Refused): + bounded_participants( + rows, Topology("main", 1, (("a", 9000), ("b", 9000))), {"a": "a", "b": "b"} + ) + + +def test_replica_duplication_does_not_consume_the_logical_identity_allowance(): + from server.core.cell_ch import Topology + from server.insights.domain.cell_participants import latest + from server.insights.store.provider_participants import bounded_participants + + rows = [ + {**participant(n), "proof_bytes": len(PROOF.encode()), "h": host} + for host in ("a", "b") + for n in range(1, 1025) + ] + selected = bounded_participants( + rows, Topology("main", 1, (("a", 9000), ("b", 9000))), {"a": "a", "b": "b"} + ) + assert len(selected) == 2048 and len(latest(selected)) == 1024 diff --git a/tests/unit/test_cell_observation_census_stream.py b/tests/unit/test_cell_observation_census_stream.py new file mode 100644 index 0000000000..0c7c967f0d --- /dev/null +++ b/tests/unit/test_cell_observation_census_stream.py @@ -0,0 +1,245 @@ +"""The complete owning reader uses real SQL and the bounded raw transport on fakes.""" + +from __future__ import annotations + +import json +import time + +import pytest + +from server.core.cell_ch import Refused +from server.insights.domain.cell_observations import Attempt +from server.insights.store import cell_observations as store +from server.settings import settings +from tests.unit.test_cell_observation_census import context, instance, observation, participant +from tests.unit.test_cell_scan import FakeCH + +HOSTS = ("os-0", "os-1") + + +def each(rows): + return [{"h": host, **row} for host in HOSTS for row in rows] + + +def fixtures(*, problem=""): + ch = FakeCH() + ch.on( + "FROM system.clusters", + [ + {"host_name": f"ch-{n}", "port": 9000, "shard_num": 1} + for n in range(65 if problem == "replica-cap" else 2) + ], + ) + ch.on("SHOW GRANTS FINAL", [{"g": "GRANT ALL ON *.* TO attach"}]) + ch.on("currentUser()", [{"u": "attach"}]) + ch.on( + "groupArrayIf(host_name, is_local)", + [{"h": host, "names": [f"ch-{n}"]} for n, host in enumerate(HOSTS)], + ) + ch.on("system.processes", each([{"user": "attach"}])) + ch.on("system.users", each([{"name": "attach"}])) + ch.on( + "system.grants", + each( + [ + { + "user_name": "attach", + "role_name": None, + "access_type": "ALL", + "database": None, + "table": None, + } + ] + ), + ) + ch.on("system.role_grants", []) + ch.on("system.row_policies", []) + ch.on("SYSTEM SYNC REPLICA", [{"host": f"ch-{n}", "status": 0} for n in range(2)]) + ch.on( + "system.replicas", + [ + { + "h": host, + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": f"r{n}", + "replica_is_active": {"r0": 1, "r1": 1}, + } + for n, host in enumerate(HOSTS) + ], + ) + return _data_routes(ch, problem) + + +def _data_routes(ch, problem): + ch.on("engine FROM", each([{"engine": "ReplicatedReplacingMergeTree"}])) + ch.on( + "engine, engine_full FROM", + each( + [ + { + "name": name, + "engine": "ReplicatedReplacingMergeTree" + if name == "_provider_participants" + else "KeeperMap", + "engine_full": "KeeperMap('/" + name + "')", + } + for name in store._TABLES + ] + ), + ) + columns = each( + [ + {"table": table, "name": name, "type": kind} + for table, cols in store._COLUMNS.items() + for name, kind in cols.items() + ] + ) + if problem == "column": + columns[0]["type"] = "String" + ch.on("system.columns", columns) + count = 0 + + def participants(sql, params): + nonlocal count + count += 1 + rows = [participant(), participant(2)] + if problem == "birth-during-join" and count >= 2: + rows.append(participant(3)) + if problem == "lower-conflict": + rows = [ + participant(version=2), + participant(version=1), + participant(state="drained", version=1), + ] + return each([{**row, "proof_bytes": len(row["host_proof"].encode())} for row in rows]) + + ch.on( + "proof_bytes FROM", + OSError("replica unavailable") if problem == "unavailable" else participants, + ) + beats = [] + for value in (observation(), observation(2, role="worker")): + raw = json.dumps(value["value"]) + beats.append( + { + "instance_id": value["instance_id"], + "cell": "later", + "cell_bytes": 5, + "tenancy_mode": "enforced", + "tenancy_mode_bytes": 8, + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "age_ms": 0, + "generation": 1, + } + ) + ch.on("`cell_heartbeats`", each(beats)) + ch.on( + "`instances`", + each( + [ + { + "instance_id": key, + **value, + "keys_fp_bytes": len(value["keys_fp"]), + "generation": 1, + } + for key, value in instance().items() + ] + ), + ) + return ch + + +async def test_complete_replica_union_reads_workers_and_rechecks_births(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch, attempt = fixtures(), Attempt() + deadline = time.monotonic() + 10 + result = await store.read_cell(ch, attempt, context(), deadline=deadline) + assert result.pending == () and result.keys_fp == "a" * 64 + assert len(result.participants) == len(result.observations) == 2 + reads = ch.sql("raw_stream") + participant_reads = [sql for sql in reads if "proof_bytes FROM" in sql] + assert len(participant_reads) == 2 and all(" FINAL" not in sql for sql in participant_reads) + beats = next(sql for sql in reads if "`cell_heartbeats`" in sql) + assert "dateDiff('millisecond', census_observation.heartbeat_at, now64(3))" in beats + assert " WHERE " not in beats and "LIMIT 1025 BY h" in beats + assert all( + bounds["skip_unavailable_shards"] == 0 + for kind, _, bounds in ch.calls + if kind == "raw_stream" + ) + retained = attempt.retained_bytes + repeated = await store.read_cell(fixtures(), attempt, context(), deadline=deadline) + assert repeated.pending == () and attempt.retained_bytes == retained + assert repeated.raw_bytes == result.raw_bytes + + +async def test_birth_during_join_becomes_an_obligation_not_an_omission(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch, attempt = fixtures(problem="birth-during-join"), Attempt() + result = await store.read_cell(ch, attempt, context(), deadline=time.monotonic() + 10) + extra = participant(3)["participant_id"] + assert extra in result.pending and extra in attempt.participants + assert len(result.participants) == 3 + + +@pytest.mark.parametrize("problem", ["replica-cap", "column", "lower-conflict", "unavailable"]) +async def test_unverified_complete_census_never_returns_a_snapshot(monkeypatch, problem): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch = fixtures(problem=problem) + with pytest.raises(OSError if problem == "unavailable" else Refused): + await store.read_cell(ch, Attempt(), context(), deadline=time.monotonic() + 10) + # A failed all-replica participant read is never repeated against just this server. + if problem == "unavailable": + reads = [sql for sql in ch.sql("raw_stream") if "proof_bytes FROM" in sql] + assert len(reads) == 1 and "clusterAllReplicas" in reads[0] + + +@pytest.mark.parametrize("field", ["oidc_client_id", "future"]) +async def test_complete_reader_accepts_matching_large_fingerprint_fields(monkeypatch, field): + from dataclasses import replace + + from server.core import cell + + monkeypatch.setattr(settings, "ch_cluster", "main") + accepted = cell.as_value({"deployment_id": "dep", field: "x" * 3000}) + assert cell.readable_fingerprint(accepted) + assert 2048 < len(accepted["fields"][field].encode()) < 64 * 1024 + configured = replace(context(), own_pair=(accepted,), directory_pair=(accepted,)) + configured.validate() + ch = fixtures() + for needle, rows in ch.routes: + if needle == "`cell_heartbeats`": + for row in rows: + value = json.loads(row["fingerprint"]) + value["fields"][field] = accepted["fields"][field] + row["fingerprint"] = json.dumps(value) + row["fingerprint_bytes"] = len(row["fingerprint"].encode()) + result = await store.read_cell(ch, Attempt(), configured, deadline=time.monotonic() + 10) + assert result.pending == () and result.keys_fp == "a" * 64 + assert { + value["value"]["attach_observation"]["role"] for value in result.observations.values() + } == {"server", "worker"} + assert all( + value["value"]["fields"][field] == accepted["fields"][field] + for value in result.observations.values() + ) + + +async def test_complete_reader_still_refuses_oversized_nonfingerprint_metadata(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + configured = context() + configured.validate() + ch = fixtures() + for needle, rows in ch.routes: + if needle == "`cell_heartbeats`": + for row in rows: + value = json.loads(row["fingerprint"]) + value["metadata_note"] = "x" * 2049 + row["fingerprint"] = json.dumps(value) + row["fingerprint_bytes"] = len(row["fingerprint"].encode()) + with pytest.raises(Refused, match="process metadata exceeds its field bound"): + await store.read_cell(ch, Attempt(), configured, deadline=time.monotonic() + 10) diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py index 353dc1e2cf..76bfb9028a 100644 --- a/tests/unit/test_cell_update_participants.py +++ b/tests/unit/test_cell_update_participants.py @@ -195,8 +195,8 @@ async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): from types import SimpleNamespace from unittest.mock import AsyncMock - from server.cli import cell_update_participants as module from server.core.cell_ch import Topology + from server.insights.store import provider_participants as module events = [] topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) @@ -250,8 +250,8 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey from types import SimpleNamespace from unittest.mock import AsyncMock - from server.cli import cell_update_participants as module from server.core.cell_ch import Topology + from server.insights.store import provider_participants as module topology = Topology( "replicas", @@ -296,6 +296,6 @@ async def per_host(*args): module.cell_sync, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) ) query = AsyncMock(side_effect=AssertionError("must not read partial census")) - with pytest.raises((module.cell_control.Refused, OSError)): + with pytest.raises((module.Refused, OSError)): await module.census(SimpleNamespace(query_all=query)) query.assert_not_awaited() diff --git a/tests/unit/test_ch_bounded_stream.py b/tests/unit/test_ch_bounded_stream.py new file mode 100644 index 0000000000..5e861d1023 --- /dev/null +++ b/tests/unit/test_ch_bounded_stream.py @@ -0,0 +1,101 @@ +"""The canonical byte reader rejects malformed or truncated raw streams.""" + +from __future__ import annotations + +import time +from contextlib import asynccontextmanager + +import pytest + +from server.core.ch_bounded import BoundedReadError, read_rows +from server.insights.domain.cell_observations import Attempt +from server.insights.store.cell_observations import _Reads + + +class Stream: + def __init__(self, chunks): + self.chunks = chunks + self.closed = 0 + + @asynccontextmanager + async def raw_stream(self, sql, params=None, *, settings=None): + async def chunks(): + for chunk in self.chunks: + yield chunk + + try: + yield chunks() + finally: + self.closed += 1 + + +@pytest.mark.parametrize( + "chunks", + [ + [b'{"x":1,"x":2}\n'], + [b'{"x":NaN}\n'], + [b'{"x":1e999}\n'], + [b'{"x":1}'], + [b'{"x":"\xff"}\n'], + [b"[]\n"], + [b"{}\n{}\n"], + ], +) +async def test_malformed_incomplete_or_extra_rows_fail_closed_and_close(chunks): + ch = Stream(chunks) + with pytest.raises(BoundedReadError): + await read_rows( + "SELECT 1", + {}, + ch=ch, + deadline=time.monotonic() + 1, + row_bytes=32, + total_bytes=64, + max_rows=1, + tag="cell.attach.census", + ) + assert ch.closed == 1 + + +async def test_chunk_boundaries_and_exact_byte_allowances_preserve_rows(): + ch = Stream([b'{"', b'x":1}', b"\n"]) + assert await read_rows( + "SELECT 1", + {}, + ch=ch, + deadline=time.monotonic() + 1, + row_bytes=7, + total_bytes=8, + max_rows=1, + tag="cell.attach.census", + ) == [{"x": 1}] + assert ch.closed == 1 + + +async def test_complete_read_aggregates_stream_bytes_but_next_poll_retains_only_obligations( + monkeypatch, +): + from server.insights.domain import cell_observations as domain + + deadline = time.monotonic() + 1 + attempt = Attempt() + monkeypatch.setattr(domain, "METADATA_BYTES", 8) + ch = Stream([b'{"x":1}\n']) + reader = _Reads(ch, attempt, deadline) + assert await reader.query_all("SELECT 1") == [{"x": 1}] + with pytest.raises(BoundedReadError): + await reader.query_all("SELECT 1") + next_poll = _Reads(ch, attempt, deadline) + assert await next_poll.query_all("SELECT 1") == [{"x": 1}] + assert attempt.deadline == deadline and next_poll.raw_bytes_used == 8 + assert attempt.retained_bytes == 0 + + +async def test_next_poll_cannot_reset_the_original_workflow_allowance(): + from server.core.cell_ch import Refused + + attempt = Attempt() + deadline = time.monotonic() + 1 + _Reads(Stream([]), attempt, deadline) + with pytest.raises(Refused, match="reset"): + _Reads(Stream([]), attempt, deadline + 1) From ec911176ff1d76f353b69db359f6a4247192a3aa Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 08:20:15 +0000 Subject: [PATCH 178/181] Test RSS census exit and unreadability races Signed-off-by: zlareb1 --- .../unit/test_cell_trigger_resource_census.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index fb3cf02058..f8df8bc60b 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -7,6 +7,7 @@ import pytest from scripts.gates.testsupport.cell_trigger import fixture +from scripts.gates.testsupport.cell_trigger.owned_children import OwnedChildren @pytest.fixture @@ -82,6 +83,67 @@ def read(path, *args, **kwargs): return owner, owned, clock +@pytest.fixture +def rss_census(census, monkeypatch): + owner, owned, clock = census + original_read, original_identity = Path.read_text, owned.identify + gone, failures = set(), {} + + def read(path, *args, **kwargs): + if path.parent.parent == Path("/proc") and path.name == "status": + pid = int(path.parent.name) + if pid in failures: + return failures[pid]() + return "Name:\tfixture\nVmRSS:\t1024 kB\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + owned.identify = lambda pid: None if pid in gone else original_identity(pid) + owned.measure = OwnedChildren.rss + return owner, owned, clock, gone, failures + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_vanished_rss_read_requires_a_new_complete_forest(rss_census, error): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise error("gone during RSS read") + + failures[300] = vanish + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_live_process_with_unreadable_rss_never_refreshes_the_sample(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS unavailable while identity remains present") + + failures[300] = unreadable + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_a_readable_rss_forest_keeps_the_existing_admission_path(rss_census): + owner, owned, _, _, _ = rss_census + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 1 + assert set(owned.sampled_pids) == {100, 200, 300} + + @pytest.mark.parametrize("stage", ["before", "after", "children"]) def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): owner, owned, _ = census From 57ca5971a7a08f693642a241ec18df8b5c0124e2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 08:32:34 +0000 Subject: [PATCH 179/181] Verify process disappearance after failed RSS reads Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/fixture.py | 8 +- .../cell_trigger/owned_children.py | 6 +- .../unit/test_cell_trigger_resource_census.py | 78 +++++++++++++++++++ 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index cadc4f9be0..1b75b43dc7 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -826,11 +826,11 @@ def complete_forest() -> bool: "resource process birth changed", ) observed[pid] = before - need( - self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], - "process RSS", - ) try: + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) children = self.owned.children(pid) except (FileNotFoundError, ProcessLookupError): need( diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py index b0c5d950f8..afafd2ece4 100644 --- a/scripts/gates/testsupport/cell_trigger/owned_children.py +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -38,10 +38,8 @@ def children(pid: int) -> list[int]: @staticmethod def rss(pid: int) -> int: - try: - status = Path("/proc", str(pid), "status").read_text() - except FileNotFoundError: - return 0 # Vanished sample, never proof that the process used no memory. + # The forest owner must verify disappearance after a failed status read. + status = Path("/proc", str(pid), "status").read_text() return next( (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 ) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index f8df8bc60b..75778534b7 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -144,6 +144,84 @@ def test_a_readable_rss_forest_keeps_the_existing_admission_path(rss_census): assert set(owned.sampled_pids) == {100, 200, 300} +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +@pytest.mark.parametrize("pid", [100, 200]) +def test_rss_root_loss_does_not_retry_or_refresh(rss_census, error, pid): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(pid) + raise error("root disappeared during RSS read") + + failures[pid] = vanish + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 1 + + +@pytest.mark.parametrize("error", [PermissionError, OSError, ValueError]) +def test_other_rss_read_errors_remain_refusals(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS read failed without verified disappearance") + + failures[300] = unreadable + owner.sample() + assert isinstance(owner.first, error) + assert owner.sampled == 7.0 + assert owned.round == 1 + + +def test_new_forest_still_checks_the_next_child_rss(rss_census): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise ProcessLookupError("gone during RSS read") + + failures[300] = vanish + failures[400] = lambda: f"VmRSS:\t{fixture.POLICY['stop_process_rss_bytes'] // 1024 + 1} kB\n" + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 2 + assert 400 in owned.sampled_pids + + +def test_rss_disappearance_does_not_reset_the_census_deadline(rss_census): + owner, owned, clock, gone, failures = rss_census + + def vanish(): + gone.add(300) + clock.now = 13.0 + raise ProcessLookupError("gone at the census deadline") + + failures[300] = vanish + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_repeated_rss_disappearance_keeps_the_three_forest_bound(rss_census): + owner, owned, _, gone, failures = rss_census + owned.trees = [[300], [400], [500], [600]] + + def vanish(pid): + gone.add(pid) + raise ProcessLookupError("another descendant vanished") + + failures.update({pid: lambda pid=pid: vanish(pid) for pid in (300, 400, 500)}) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 3 + assert 600 not in owned.sampled_pids + + @pytest.mark.parametrize("stage", ["before", "after", "children"]) def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): owner, owned, _ = census From f0612d3c115b810c9bfcfc819ec22183860c0744 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 08:55:52 +0000 Subject: [PATCH 180/181] Keep placement test setup outside the HTTP deadline Signed-off-by: zlareb1 --- tests/unit/test_cell_place_http_deadline.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_cell_place_http_deadline.py b/tests/unit/test_cell_place_http_deadline.py index 79d278903a..56763a5bdc 100644 --- a/tests/unit/test_cell_place_http_deadline.py +++ b/tests/unit/test_cell_place_http_deadline.py @@ -14,6 +14,7 @@ from server.cli import cell_control, cell_place from server.settings import settings from tests.unit import test_cell_place as helpers +from tests.unit.clock_fake import ManualClock local = helpers.local invoke_place = helpers.invoke_place @@ -24,6 +25,10 @@ @pytest.fixture def exchange(local, monkeypatch): conn, _ = local + # The work timer stays real; scheduler delay cannot spend the fake close budget. + phase_clock = ManualClock() + monkeypatch.setattr(cell_control, "time", phase_clock) + monkeypatch.setattr(cell_control.pg, "time", phase_clock) events = [] entered = asyncio.Event() state = {"mode": "response", "reads": [], "owners": []} @@ -93,9 +98,11 @@ async def __aexit__(self, *args): raise return await super().__aexit__(*args) + # TLS/pool setup is not part of the inert exchange being timed. + transport = httpx.AsyncHTTPTransport() + transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) + def client(**kwargs): - transport = httpx.AsyncHTTPTransport() - transport._pool = httpcore.AsyncConnectionPool(network_backend=Backend([])) return Client(transport=transport, **kwargs) conn.close = close From 3f6d7646d1813b8a07b01afe83a98c36ff440eba Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 19:20:54 +0000 Subject: [PATCH 181/181] Compare complete known observation fingerprints Apply known defaults before checking accepted reservations. Keep the shared transaction allowance at its owning imports and preserve exact census validation while correcting inferred types. Signed-off-by: zlareb1 --- server/cli/cell_prepare.py | 3 +- server/cli/cell_update_participants.py | 6 +-- server/cli/cell_update_probe.py | 7 ++-- server/insights/domain/cell_observations.py | 10 ++--- server/insights/store/cell_observations.py | 4 +- .../insights/store/provider_participants.py | 8 ++-- tests/unit/test_cell_control_census.py | 2 +- tests/unit/test_cell_observation_census.py | 39 +++++++++++++++++++ 8 files changed, 61 insertions(+), 18 deletions(-) diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index eac6638fd1..0a3a1138a9 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -41,6 +41,7 @@ from server.cli import cell_control from server.cli._ch import writes from server.core import cell +from server.core.cell_ch import TRANSACTION_S from server.settings import settings # How long a run retries a busy or unreachable probe listener by default. @@ -548,7 +549,7 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: current, nxt = _own_rows(rows, snapshot, every) written = await owner.call( lambda: _write_rows(conn, rows, answer, current, nxt), - total_s=cell_control.TRANSACTION_S, + total_s=TRANSACTION_S, ) print( json.dumps( diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index 3e8afbc6af..2a88e631ca 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -11,7 +11,7 @@ from server.cli import cell_control from server.core import cell, instances -from server.core.cell_ch import complete +from server.core.cell_ch import TRANSACTION_S, complete from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -121,7 +121,7 @@ async def replacement( async def settle(run: Any, target: dict[str, Any]) -> None: """Fix one complete captured set; expiry never removes an obligation.""" with query_tag("cell.update.participants"): - initial = await run.owner.call(lambda: census(run.ch), total_s=cell_control.TRANSACTION_S) + initial = await run.owner.call(lambda: census(run.ch), total_s=TRANSACTION_S) captured = capture(list(initial.values())) done: set[str] = set() deadline = time.monotonic() + run.args.wait @@ -135,7 +135,7 @@ async def poll() -> bool: done.add(identity) return await replacement(run.ch, run, rows, set(captured), target) - live = await run.owner.call(poll, total_s=cell_control.TRANSACTION_S) + live = await run.owner.call(poll, total_s=TRANSACTION_S) remaining = sorted(set(captured) - done) if not remaining and live: return diff --git a/server/cli/cell_update_probe.py b/server/cli/cell_update_probe.py index d102969225..47aaf45ce1 100644 --- a/server/cli/cell_update_probe.py +++ b/server/cli/cell_update_probe.py @@ -6,6 +6,7 @@ from typing import Any from server.cli import cell_control, cell_prepare +from server.core.cell_ch import TRANSACTION_S from server.router.app import REGISTER_ANSWER_MAX_S # Read/probe/commit bounds already used by prepare, plus its reply margin. @@ -21,7 +22,7 @@ async def clean() -> None: try: await run.owner.call( - lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=cell_control.TRANSACTION_S + lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=TRANSACTION_S ) async with asyncio.timeout(run.args.wait + ANSWER_S): answer = await cell_prepare._register( @@ -36,9 +37,9 @@ async def clean() -> None: cell_control.preconditions(answer) except BaseException: try: - await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + await run.owner.call(clean, total_s=TRANSACTION_S) except BaseException: # noqa: BLE001 — preserve the original operation failure. pass raise - await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + await run.owner.call(clean, total_s=TRANSACTION_S) return answer diff --git a/server/insights/domain/cell_observations.py b/server/insights/domain/cell_observations.py index 0946d607ee..1af729220e 100644 --- a/server/insights/domain/cell_observations.py +++ b/server/insights/domain/cell_observations.py @@ -34,15 +34,15 @@ def text(value: Any, *, empty: bool = False) -> str: def identity(value: Any, *, instance: bool = False) -> str: - value = text(value) + canonical = text(value) try: - parsed = UUID(value) + parsed = UUID(canonical) except ValueError: raise refused("unreadable process identity") from None expected = parsed.hex if instance else str(parsed) - if value != expected: + if canonical != expected: raise refused("noncanonical process identity") - return value + return canonical def _size(value: Any) -> int: @@ -255,7 +255,7 @@ def _admitted(context: Context, value: dict[str, Any]) -> dict[str, Any]: claim = _claim(value) if value["cell"] != context.cell_name or value["mode"] != "enforced": raise refused("process observation belongs to another or unenforced cell") - fields = value["value"]["fields"] + fields = {**cell.DEFAULTS, **value["value"]["fields"]} for pair in (context.own_pair, context.directory_pair): if not any(cell.matches(fields, accepted) for accepted in pair): raise refused("process fingerprint disagrees with an accepted pair") diff --git a/server/insights/store/cell_observations.py b/server/insights/store/cell_observations.py index 8d3a8a3004..2c1c837f21 100644 --- a/server/insights/store/cell_observations.py +++ b/server/insights/store/cell_observations.py @@ -333,6 +333,7 @@ def _collapse_instances(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]] for row in rows: key = domain.identity(row.get("instance_id"), instance=True) _projected_field(row, "keys_fp") + generation = row.get("generation") value = { "keys_fp": domain.text(row.get("keys_fp"), empty=True), **{name: row.get(name) for name in ("serving", "stopped", "age_ms", "generation")}, @@ -344,7 +345,8 @@ def _collapse_instances(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]] ) or value["serving"] not in (0, 1) or value["stopped"] not in (0, 1) - or value["generation"] < 0 + or type(generation) is not int + or generation < 0 ): raise refused("unreadable web instance metadata") replica = (row["h"], key) diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index 34c907b0ba..7e658b7a81 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -296,7 +296,7 @@ async def census( ): raise refused("participant table has an unsupported replacement policy") if engine.startswith("Replicated"): - rows = await per_host( + replica_rows = await per_host( ch, topology, names, @@ -307,12 +307,12 @@ async def census( ), {"db": database}, ) - if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( - row for value in rows.values() for row in value + if any(len(value) != 1 for value in replica_rows.values()) or cell_catalog.extra_replicas( + row for value in replica_rows.values() for row in value ): raise refused("participant replica set includes an unverified replica") if bounded: - bounded_replica_paths(rows) + bounded_replica_paths(replica_rows) budget = cell_sync.Budget(TRANSACTION_S) await cell_sync._system( ch, diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index b80384b90e..aaed352e13 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -365,7 +365,7 @@ def function(path: str, name: str) -> ast.AST: "_prepare_locked", [ "cell_control.claim(conn, call, snapshot)", - "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=cell_control.TRANSACTION_S)", + "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=TRANSACTION_S)", ], ), ( diff --git a/tests/unit/test_cell_observation_census.py b/tests/unit/test_cell_observation_census.py index d364a807ef..fce91e471b 100644 --- a/tests/unit/test_cell_observation_census.py +++ b/tests/unit/test_cell_observation_census.py @@ -293,3 +293,42 @@ def test_replica_duplication_does_not_consume_the_logical_identity_allowance(): rows, Topology("main", 1, (("a", 9000), ("b", 9000))), {"a": "a", "b": "b"} ) assert len(selected) == 2048 and len(latest(selected)) == 1024 + + +@pytest.mark.parametrize( + ("field", "nondefault"), + [("deployment_id", "dep"), ("public_host", "cell.example.test"), ("configured_orgs", ["acme"])], +) +@pytest.mark.parametrize("pair", ["own", "directory"]) +def test_omitted_observation_field_cannot_match_a_nondefault_pair(field, nondefault, pair): + row, value = participant(), observation(role="worker") + value["value"]["fields"] = {} + default = cell.as_value(cell.DEFAULTS) + accepted = cell.as_value({field: nondefault}) + context = domain.Context( + "later", + False, + (accepted if pair == "own" else default,), + (accepted if pair == "directory" else default,), + KEYS, + ) + attempt = domain.Attempt() + rows = {row["participant_id"]: row} + attempt.capture(rows) + attempt.retain_observation(value) + with pytest.raises(Refused, match="fingerprint disagrees"): + attempt.judge(context, rows, {value["instance_id"]: value}, {}) + + +def test_omitted_default_fields_preserve_unknown_observation_fields(): + row, value = participant(), observation(role="worker") + fields = {"future_field": {"enabled": True}} + value["value"]["fields"] = copy.deepcopy(fields) + accepted = cell.as_value({**cell.DEFAULTS, **fields}) + context = domain.Context("later", False, (accepted,), (accepted,), KEYS) + attempt = domain.Attempt() + rows = {row["participant_id"]: row} + attempt.capture(rows) + attempt.retain_observation(value) + assert attempt.judge(context, rows, {value["instance_id"]: value}, {}) == ((), KEYS) + assert value["value"]["fields"] == fields