diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c6e07484b..fe8d32c23f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1016,6 +1016,7 @@ jobs: tests/integration/test_cell_update_addresses.py tests/integration/test_cell_update_run_pg.py tests/integration/test_cell_update_participants_ch.py + tests/integration/test_cell_observation_census.py tests/integration/test_cell_place_cleanup.py tests/integration/test_cell_attach_two_replicas.py tests/integration/test_cell_attach_cleanup.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index ff46557fb8..6403bca6f2 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1654,6 +1654,30 @@ before any cell's processes do, since a prepared first cell asks the state route first cell-state read until it reads `serving`, and only a router of that release answers it from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). +## Bounded process census + +The shared process reader has an explicit bounded mode for complete process evidence. +It reads permanent participants, every stored cell heartbeat and every stored web +instance across the configured replicas, then reads participants again. Replicated +participant tables synchronize before selection; shared tables use sequential reads. +Catalog, membership, privilege or conflicting-version uncertainty refuses the read. +The ordinary update reader keeps its existing query and allowance. + +Each complete read shares an 8 MiB raw allowance across its streams. A caller keeps one +attempt across polls and physical rounds: at most 1024 participant identities, 1024 +observation identities and 8 MiB retained metadata, including certified retirements. +Repeated evidence replaces bounded latest state without evicting captured obligations. +The reader binds the original monotonic deadline and accepts at most 64 configured +replicas. Oversized, truncated or malformed results refuse rather than yield a subset. + +Supported process observations bind the actual participant, certified birth, cell, +accepted fingerprints and root-key fingerprint. Workers join without web instance rows; +traffic-serving first-cell servers also need their router marker. Missing, stale, +future or nonaccepting observations remain unresolved obligations. Native commands +without observations require positive retirement; this component grants no owner +exemption. Its internal result carries unresolved identities and selected evidence. +It performs no attach, exposes no diagnostic route and certifies no physical isolation. + ## Invariants - An org is served by exactly one cell, the one its placement names, and its placement diff --git a/scripts/gates/testsupport/cell_trigger/fixture.py b/scripts/gates/testsupport/cell_trigger/fixture.py index cadc4f9be0..1b75b43dc7 100644 --- a/scripts/gates/testsupport/cell_trigger/fixture.py +++ b/scripts/gates/testsupport/cell_trigger/fixture.py @@ -826,11 +826,11 @@ def complete_forest() -> bool: "resource process birth changed", ) observed[pid] = before - need( - self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], - "process RSS", - ) try: + need( + self.owned.rss(pid) * 1024 <= POLICY["stop_process_rss_bytes"], + "process RSS", + ) children = self.owned.children(pid) except (FileNotFoundError, ProcessLookupError): need( diff --git a/scripts/gates/testsupport/cell_trigger/owned_children.py b/scripts/gates/testsupport/cell_trigger/owned_children.py index b0c5d950f8..afafd2ece4 100644 --- a/scripts/gates/testsupport/cell_trigger/owned_children.py +++ b/scripts/gates/testsupport/cell_trigger/owned_children.py @@ -38,10 +38,8 @@ def children(pid: int) -> list[int]: @staticmethod def rss(pid: int) -> int: - try: - status = Path("/proc", str(pid), "status").read_text() - except FileNotFoundError: - return 0 # Vanished sample, never proof that the process used no memory. + # The forest owner must verify disappearance after a failed status read. + status = Path("/proc", str(pid), "status").read_text() return next( (int(line.split()[1]) for line in status.splitlines() if line.startswith("VmRSS:")), 0 ) diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 53d65b7838..e799f00213 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -14,13 +14,11 @@ from server.cli import cell_attach_pg as apg from server.core import pg, retirement -from server.core.cell_ch import Refusal +from server.core.cell_ch import TRANSACTION_S, Refusal from server.core.cell_ch import Refused as Refused from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings -# A control transaction has few metadata statements; COMMIT's standby wait is bounded too. -TRANSACTION_S = 60.0 # not a setting: a single control transaction's deadline OPEN_S = 75.0 # not a setting: connection initialization and direct-session checks REQUEST_S = CONTROL_MAX_S + 30.0 # not a setting: directory bound plus probes/connect/close Call = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] diff --git a/server/cli/cell_prepare.py b/server/cli/cell_prepare.py index eac6638fd1..0a3a1138a9 100644 --- a/server/cli/cell_prepare.py +++ b/server/cli/cell_prepare.py @@ -41,6 +41,7 @@ from server.cli import cell_control from server.cli._ch import writes from server.core import cell +from server.core.cell_ch import TRANSACTION_S from server.settings import settings # How long a run retries a busy or unreachable probe listener by default. @@ -548,7 +549,7 @@ async def call(path: str, body: dict[str, Any]) -> dict[str, Any]: current, nxt = _own_rows(rows, snapshot, every) written = await owner.call( lambda: _write_rows(conn, rows, answer, current, nxt), - total_s=cell_control.TRANSACTION_S, + total_s=TRANSACTION_S, ) print( json.dumps( diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index b948d47e8e..2a88e631ca 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -10,8 +10,8 @@ from uuid import UUID from server.cli import cell_control -from server.core import cell, cell_sync, ch_names, instances -from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology +from server.core import cell, instances +from server.core.cell_ch import TRANSACTION_S, complete from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -19,8 +19,9 @@ participant_tag, proves_stopped, ) -from server.insights.store import cell_catalog +from server.insights.domain.cell_participants import latest from server.insights.store import provider_participants as participants +from server.insights.store.provider_participants import census class CompleteReads: @@ -36,44 +37,6 @@ async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: ) -def complete(sql: str, *, sequential: int = 0) -> str: - return ( - sql - + " " - + WHOLE_SQL - + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " - + f"select_sequential_consistency={sequential}" - ) - - -def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - out: dict[str, dict[str, Any]] = {} - # The latest winner cannot hide a conflicting older version from another replica. - seen: dict[tuple[str, int], tuple[str, str]] = {} - for row in rows: - try: - identity = str(UUID(str(row["participant_id"]))) - version = row["version"] - state, proof = row["state"], row["host_proof"] - except (KeyError, TypeError, ValueError): - raise cell_control._refused("participant census contains an unreadable row") from None - if ( - type(version) is not int - or version <= 0 - or state not in ("running", "drained", "retired") - or not isinstance(proof, str) - ): - raise cell_control._refused("participant census contains an unreadable row") - previous = out.get(identity) - observed = seen.get((identity, version)) - if observed is not None and observed != (state, proof): - raise cell_control._refused(f"participant {identity} has conflicting equal versions") - seen[(identity, version)] = (state, proof) - if previous is None or int(previous["version"]) < version: - out[identity] = dict(row) - return out - - def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: return { identity: row @@ -82,70 +45,6 @@ def capture(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: } -async def census(ch: Any) -> dict[str, dict[str, Any]]: - """Synchronize after the durable cutoff; every configured replica must answer.""" - topology = await read_topology(ch) - if topology.cluster and not topology.entries: - raise cell_control._refused("participant census cannot resolve the configured cluster") - if problems := topology.refusals(): - raise Refused(problems) - names = await cell_catalog.host_names(ch, topology) if topology.multi else {} - if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: - raise Refused(problems) - database = ch_names.database() - tables = await per_host( - ch, - topology, - names, - ( - "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " - "name='_provider_participants'" - ), - {"db": database}, - ) - if not tables or any(len(rows) != 1 for rows in tables.values()): - raise cell_control._refused("participant table is missing from a configured replica") - engines = {str(rows[0]["engine"]) for rows in tables.values()} - if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): - raise cell_control._refused( - "participant replicas have unreadable or different table engines" - ) - engine = next(iter(engines)) - if engine.startswith("Replicated"): - rows = await per_host( - ch, - topology, - names, - ( - "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " - "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " - "table='_provider_participants'" - ), - {"db": database}, - ) - if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( - row for value in rows.values() for row in value - ): - raise cell_control._refused("participant replica set includes an unverified replica") - budget = cell_sync.Budget(cell_control.TRANSACTION_S) - await cell_sync._system( - ch, - f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " - f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", - topology, - budget, - "participant census", - ) - source = f"{quote(database)}.`_provider_participants`" - params = {} - if topology.multi: - source = f"clusterAllReplicas({{cluster:String}}, {source})" - params["cluster"] = topology.cluster - sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" - rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) - return latest(rows) - - async def stopped(ch: Any, initial: dict[str, Any], current: dict[str, Any] | None) -> bool: if current is not None: try: @@ -222,7 +121,7 @@ async def replacement( async def settle(run: Any, target: dict[str, Any]) -> None: """Fix one complete captured set; expiry never removes an obligation.""" with query_tag("cell.update.participants"): - initial = await run.owner.call(lambda: census(run.ch), total_s=cell_control.TRANSACTION_S) + initial = await run.owner.call(lambda: census(run.ch), total_s=TRANSACTION_S) captured = capture(list(initial.values())) done: set[str] = set() deadline = time.monotonic() + run.args.wait @@ -236,7 +135,7 @@ async def poll() -> bool: done.add(identity) return await replacement(run.ch, run, rows, set(captured), target) - live = await run.owner.call(poll, total_s=cell_control.TRANSACTION_S) + live = await run.owner.call(poll, total_s=TRANSACTION_S) remaining = sorted(set(captured) - done) if not remaining and live: return diff --git a/server/cli/cell_update_pg.py b/server/cli/cell_update_pg.py index 9be29d2571..321c6c38dd 100644 --- a/server/cli/cell_update_pg.py +++ b/server/cli/cell_update_pg.py @@ -8,7 +8,6 @@ from server.cli import cell_control from server.core import cell -from server.router import fingerprints @dataclass(frozen=True) @@ -48,8 +47,8 @@ async def read(conn: Any, name: str) -> State: "update needs complete prepared state naming this cell; prepare or restore it first" ) current, pending = cell.json_object(row["current_fp"]), cell.json_object(row["next_fp"]) - if not fingerprints.readable(current) or ( - pending is not None and not fingerprints.readable(pending) + if not cell.readable_fingerprint(current) or ( + pending is not None and not cell.readable_fingerprint(pending) ): raise cell_control._refused("update needs readable local current/next fingerprints") assert current is not None diff --git a/server/cli/cell_update_probe.py b/server/cli/cell_update_probe.py index d102969225..47aaf45ce1 100644 --- a/server/cli/cell_update_probe.py +++ b/server/cli/cell_update_probe.py @@ -6,6 +6,7 @@ from typing import Any from server.cli import cell_control, cell_prepare +from server.core.cell_ch import TRANSACTION_S from server.router.app import REGISTER_ANSWER_MAX_S # Read/probe/commit bounds already used by prepare, plus its reply margin. @@ -21,7 +22,7 @@ async def clean() -> None: try: await run.owner.call( - lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=cell_control.TRANSACTION_S + lambda: cell_prepare._write_nonces(run.ch, nonces), total_s=TRANSACTION_S ) async with asyncio.timeout(run.args.wait + ANSWER_S): answer = await cell_prepare._register( @@ -36,9 +37,9 @@ async def clean() -> None: cell_control.preconditions(answer) except BaseException: try: - await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + await run.owner.call(clean, total_s=TRANSACTION_S) except BaseException: # noqa: BLE001 — preserve the original operation failure. pass raise - await run.owner.call(clean, total_s=cell_control.TRANSACTION_S) + await run.owner.call(clean, total_s=TRANSACTION_S) return answer diff --git a/server/core/cell.py b/server/core/cell.py index eb996a5fcc..9de44e23ec 100644 --- a/server/core/cell.py +++ b/server/core/cell.py @@ -240,6 +240,52 @@ def matches(mine: Mapping[str, Any], value: Mapping[str, Any] | None) -> bool: return True +def readable_fingerprint(value: Any) -> bool: + """Known fields have exact types; unknown JSON fields remain reserved and intact.""" + if not isinstance(value, dict) or set(value) != {"v", "fields"}: + return False + if type(value["v"]) is not int or value["v"] != FINGERPRINT_VERSION: + return False + fields = value["fields"] + if not isinstance(fields, dict): + return False + try: + # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + return False + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + try: + if "\0" in item: + return False + item.encode() + except UnicodeError: + return False + elif isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + return False + pending.extend(item.keys()) + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + elif item is not None and type(item) not in (bool, int, float): + return False + for name, default in DEFAULTS.items(): + item = fields.get(name, default) + if type(item) is not type(default): + return False + if isinstance(item, list) and any(not isinstance(part, str) for part in item): + return False + for name in ("configured_orgs", "end_user_orgs"): + for org in fields.get(name, []): + if named_org(org) is None or org_name_refusal(org): + return False + return True + + def digest(fields: Mapping[str, Any]) -> str: """A short digest of a fingerprint for the gate's reason; never compared.""" blob = json.dumps(dict(fields), sort_keys=True, separators=(",", ":")) diff --git a/server/core/cell_ch.py b/server/core/cell_ch.py index 6262a23176..84873b609d 100644 --- a/server/core/cell_ch.py +++ b/server/core/cell_ch.py @@ -13,6 +13,9 @@ from server.core.ch import query_tag from server.settings import settings +# A control transaction includes the standby acknowledgment of COMMIT. +TRANSACTION_S = 60.0 # not a setting: one control transaction's deadline + # system.query_log's log_comment: the scan's, and the writer check's. Attach's own grant # reads carry the tag of the check that runs them; the host names always carry the scan's. TAG = "cell.scan" @@ -130,16 +133,19 @@ def refusals(self) -> list[Refusal]: return out -async def read_topology(ch: Any) -> Topology: +async def read_topology(ch: Any, *, max_replicas: int | None = None) -> Topology: cluster = settings.ch_cluster if not cluster: return Topology("", 0) with query_tag(TAG): - rows = await ch.query_all( - "SELECT host_name, port, shard_num FROM system.clusters WHERE cluster = {c:String} " - + WHOLE_SQL, - {"c": cluster}, - ) + sql = "SELECT host_name, port, shard_num FROM system.clusters WHERE cluster = {c:String} " + params: dict[str, Any] = {"c": cluster} + if max_replicas is not None: + sql += "LIMIT {replica_limit:UInt16} " + params["replica_limit"] = max_replicas + 1 + rows = await ch.query_all(sql + WHOLE_SQL, params) + if max_replicas is not None: + bounded_topology(rows, max_replicas) return Topology( cluster, len({int(r["shard_num"]) for r in rows}), @@ -175,3 +181,30 @@ async def per_host( raise Refused([Refusal(f"host {r['h']}", "answered without a configured name")]) out[name].append(r) return out + + +def complete(sql: str, *, sequential: int = 0) -> str: + return ( + sql + + " " + + WHOLE_SQL + + ", skip_unavailable_shards=0, prefer_localhost_replica=1, " + + f"select_sequential_consistency={sequential}" + ) + + +def bounded_topology(rows: list[dict[str, Any]], limit: int) -> None: + """Reject cap+1 and malformed topology before Topology builds sets or maps.""" + if len(rows) > limit: + raise Refused([Refusal("participant census", "configured replica limit exceeded")]) + for row in rows: + if ( + not isinstance(row.get("host_name"), str) + or not row["host_name"] + or len(row["host_name"].encode()) > 2048 + or type(row.get("port")) is not int + or not 0 < row["port"] <= 65535 + or type(row.get("shard_num")) is not int + or row["shard_num"] <= 0 + ): + raise Refused([Refusal("participant census", "unreadable configured replica")]) diff --git a/server/core/ch_bounded.py b/server/core/ch_bounded.py new file mode 100644 index 0000000000..4799b99b19 --- /dev/null +++ b/server/core/ch_bounded.py @@ -0,0 +1,112 @@ +"""Finite JSONEachRow reads with limits enforced before decoding rows.""" + +from __future__ import annotations + +import asyncio +import json +import math +import time +from typing import Any + +from server.core.ch import own_write_reads, set_query_tag, statement_timeout + + +class BoundedReadError(ValueError): + """The complete answer cannot be established within its read allowance.""" + + +def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise BoundedReadError("Duplicate result field") + result[key] = value + return result + + +def _constant(value: str) -> Any: + raise BoundedReadError("Nonfinite result value") + + +def _float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise BoundedReadError("Nonfinite result value") + return result + + +def decode_json(text: str) -> Any: + """Decode already byte-bounded JSON without duplicate keys or nonfinite numbers.""" + try: + return json.loads( + text, + object_pairs_hook=_object, + parse_constant=_constant, + parse_float=_float, + ) + except (UnicodeError, ValueError, RecursionError) as error: + raise BoundedReadError("Malformed JSON value") from error + + +def _row(line: bytes) -> dict[str, Any]: + try: + row = decode_json(line.decode("utf-8")) + except UnicodeError as error: + raise BoundedReadError("Result row is not UTF-8") from error + if not isinstance(row, dict): + raise BoundedReadError("Result row is not an object") + return row + + +async def read_rows( + sql: str, + params: dict[str, Any], + *, + ch: Any, + deadline: float, + row_bytes: int, + total_bytes: int, + max_rows: int, + tag: str, + max_read_rows: int = 1_000_000, + max_read_bytes: int = 128 * 1024**2, +) -> list[dict[str, Any]]: + """Read a caller's fixed SELECT with its explicit LIMIT and JSONEachRow format.""" + remaining = min(2.0, deadline - time.monotonic()) + if remaining <= 0 or min(row_bytes, total_bytes, max_rows, max_read_rows, max_read_bytes) <= 0: + raise BoundedReadError("Read allowance exhausted") + limits = { + "max_execution_time": remaining, + "max_memory_usage": 128 * 1024**2, + "max_rows_to_read": min(max_read_rows, 1_000_000), + "max_bytes_to_read": min(max_read_bytes, 128 * 1024**2), + "max_result_rows": max_rows + 1, + "max_result_bytes": total_bytes + 1, + "read_overflow_mode": "throw", + "result_overflow_mode": "throw", + "timeout_overflow_mode": "throw", + "output_format_json_quote_64bit_integers": 0, + } + rows: list[dict[str, Any]] = [] + pending = bytearray() + consumed = 0 + set_query_tag(tag) + async with asyncio.timeout(remaining): + with own_write_reads(), statement_timeout(remaining): + async with ch.raw_stream(sql, params=params, settings=limits) as chunks: + async for chunk in chunks: + consumed += len(chunk) + if consumed > total_bytes: + raise BoundedReadError("Result exceeds its byte bound") + pending.extend(chunk) + while (end := pending.find(b"\n")) >= 0: + if end > row_bytes or len(rows) >= max_rows: + raise BoundedReadError("Result exceeds its row bound") + line = bytes(pending[:end]) + del pending[: end + 1] + rows.append(_row(line)) + if len(pending) > row_bytes: + raise BoundedReadError("Row exceeds its byte bound") + if pending: + raise BoundedReadError("Incomplete result row") + return rows diff --git a/server/insights/domain/cell_observations.py b/server/insights/domain/cell_observations.py new file mode 100644 index 0000000000..1af729220e --- /dev/null +++ b/server/insights/domain/cell_observations.py @@ -0,0 +1,294 @@ +"""Finite process observations and cumulative obligations for one cell attempt.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any +from uuid import UUID + +from server.core import cell +from server.core.provider_process import certified, has_certificate +from server.insights.domain.cell_participants import bounded_participant, refused + +PARTICIPANTS = 1024 +OBSERVATIONS = 1024 +REPLICAS = 64 +METADATA_BYTES = 8 * 1024**2 +FINGERPRINT_BYTES = 256 * 1024 +FIELD_BYTES = 2048 + + +def text(value: Any, *, empty: bool = False) -> str: + if not isinstance(value, str) or (not empty and not value): + raise refused("unreadable process metadata") + try: + if "\0" in value or len(value.encode()) > FIELD_BYTES: + raise ValueError + except UnicodeError: + raise refused("unreadable process metadata") from None + except ValueError: + raise refused("process metadata exceeds its field bound") from None + return value + + +def identity(value: Any, *, instance: bool = False) -> str: + canonical = text(value) + try: + parsed = UUID(canonical) + except ValueError: + raise refused("unreadable process identity") from None + expected = parsed.hex if instance else str(parsed) + if canonical != expected: + raise refused("noncanonical process identity") + return canonical + + +def _size(value: Any) -> int: + return len( + json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode() + ) + + +@dataclass(frozen=True) +class Context: + """Already bounded own/directory values, supplied by the normal owning caller.""" + + cell_name: str + is_first: bool + own_pair: tuple[dict[str, Any], ...] + directory_pair: tuple[dict[str, Any], ...] + recorded_keys_fp: str | None + + def validate(self) -> None: + if not isinstance(self.cell_name, str) or not cell.NAME.fullmatch(self.cell_name): + raise refused("unreadable census cell") + if type(self.is_first) is not bool: + raise refused("unreadable census first-cell state") + for pair in (self.own_pair, self.directory_pair): + if not 1 <= len(pair) <= 2 or any(not cell.readable_fingerprint(v) for v in pair): + raise refused("unreadable accepted fingerprint pair") + if _size(pair) > 64 * 1024: + raise refused("accepted fingerprint pair exceeds its bound") + if self.recorded_keys_fp is not None and not cell.SHA256.fullmatch(self.recorded_keys_fp): + raise refused("unreadable recorded root-key fingerprint") + + +def _claim(observed: dict[str, Any]) -> dict[str, Any]: + value = observed["value"] + claim = value.get("attach_observation") + keys = {"protocol", "role", "participant_id", "host_proof", "keys_fp", "accepting", "period_ms"} + if not isinstance(claim, dict) or set(claim) != keys: + raise refused("process reports no supported attach observation") + if type(claim["protocol"]) is not int or claim["protocol"] != 1: + raise refused("process reports an unsupported observation protocol") + if claim["role"] not in ("server", "worker") or type(claim["accepting"]) is not bool: + raise refused("process reports an unreadable observation role or acceptance") + if identity(claim["participant_id"]) != observed["participant_id"]: + raise refused("inner and outer participant identities disagree") + proof = text(claim["host_proof"]) + if not has_certificate(proof): + raise refused("process reports no installed lifecycle certificate") + keys_fp = text(claim["keys_fp"]) + if not cell.SHA256.fullmatch(keys_fp): + raise refused("process reports an unreadable root-key fingerprint") + if type(claim["period_ms"]) is not int or not 1000 <= claim["period_ms"] <= 60000: + raise refused("process reports an unsupported heartbeat period") + if type(value.get("serves")) is not bool or type(value.get("marked")) is not bool: + raise refused("process reports unreadable serving or marker state") + if claim["role"] == "worker" and value["serves"]: + raise refused("worker observation claims to serve traffic") + return claim + + +@dataclass +class Attempt: + """One invocation retains first obligations and latest evidence without eviction.""" + + participants: dict[str, dict[str, Any]] = field(default_factory=dict) + observations: dict[str, dict[str, Any]] = field(default_factory=dict) + _sizes: dict[tuple[str, str], int] = field(default_factory=dict) + retained_bytes: int = 0 + deadline: float | None = None + cell_name: str | None = None + + def _retain(self, kind: str, key: str, value: dict[str, Any]) -> None: + table = self.participants if kind == "participant" else self.observations + limit = PARTICIPANTS if kind == "participant" else OBSERVATIONS + if key not in table and len(table) >= limit: + raise refused(f"cumulative {kind} identity limit exceeded") + size = _size(value) + total = self.retained_bytes - self._sizes.get((kind, key), 0) + size + if total > METADATA_BYTES: + raise refused("cumulative process metadata byte limit exceeded") + table[key] = value + self._sizes[(kind, key)] = size + self.retained_bytes = total + + def capture(self, rows: dict[str, dict[str, Any]]) -> None: + for key, raw in rows.items(): + current = bounded_participant(raw) + if key != current["participant_id"]: + raise refused("participant census key disagrees with its row") + birth = certified(current["host_proof"]) + previous = self.participants.get(key) + if previous: + if previous["birth"] != birth: + raise refused("participant UUID is reused across process births") + latest = previous["latest"] + if current["version"] < latest["version"]: + raise refused("participant version regressed") + if current["version"] == latest["version"] and current != latest: + raise refused("participant has conflicting equal versions across reads") + if previous["closed"] and current["state"] != "retired": + raise refused("a certified retired participant re-enrolled") + closed = current["state"] == "retired" and has_certificate(current["host_proof"]) + first = ( + previous["first"] if previous else {"birth": birth, "version": current["version"]} + ) + self._retain( + "participant", + key, + {"first": first, "latest": current, "birth": birth, "closed": closed}, + ) + + def retain_observation(self, observed: dict[str, Any]) -> None: + key = observed["instance_id"] + previous = self.observations.get(key) + participant_id = observed["participant_id"] + participant = self.participants.get(participant_id) + if participant is None: + raise refused("observation has no participant in the complete census") + # A retired historical row need not acquire an observation capability retroactively. + if not participant["closed"]: + claim = _claim(observed) + if claim["host_proof"] != participant["birth"]: + raise refused("observation and participant process births disagree") + digest = hashlib.sha256( + json.dumps( + {k: v for k, v in observed.items() if k not in {"age_ms", "generation"}}, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + if previous: + if previous["participant_id"] != participant_id: + raise refused("observation identity is reused by another participant") + if observed["generation"] < previous["generation"]: + raise refused("observation generation regressed") + if observed["generation"] == previous["generation"] and digest != previous["digest"]: + raise refused("observation has conflicting equal generations") + for other_key, other in self.observations.items(): + if other_key != key and other["participant_id"] == participant_id: + raise refused("participant reused its observation identity") + # Retain identity/generation evidence, not a history of fingerprint payloads. + self._retain( + "observation", + key, + { + "participant_id": participant_id, + "generation": observed["generation"], + "digest": digest, + }, + ) + + def judge( + self, + context: Context, + current: dict[str, dict[str, Any]], + observed: dict[str, dict[str, Any]], + instances: dict[str, dict[str, Any]], + ) -> tuple[tuple[str, ...], str | None]: + """Return unresolved obligations, never a diagnostic or attach-success flag.""" + context.validate() + pending: set[str] = set() + by_participant: dict[str, dict[str, Any]] = {} + keys: set[str] = set() + for value in observed.values(): + participant = value["participant_id"] + if participant in by_participant: + raise refused("participant has more than one process observation") + by_participant[participant] = value + if self.participants[participant]["closed"]: + continue + claim = _admitted(context, value) + if not claim["accepting"] or not 0 <= value["age_ms"] <= 3 * claim["period_ms"]: + pending.add(participant) + if not _instance_join(value, claim, instances): + pending.add(participant) + keys.add(claim["keys_fp"]) + for key, participant in self.participants.items(): + if not participant["closed"] and ( + key not in current + or key not in by_participant + or not has_certificate(current[key]["host_proof"]) + ): + pending.add(key) + for key, instance in instances.items(): + if not instance["stopped"]: + joined = observed.get(key) + if joined is None: + pending.add("instance:" + key) + elif ( + self.participants[joined["participant_id"]]["closed"] + and instance["age_ms"] <= 180000 + ): + pending.add("instance:" + key) + if len(keys) > 1: + raise refused("processes report different root-key fingerprints") + actual_keys = next(iter(keys)) if keys else None + if ( + actual_keys + and context.recorded_keys_fp is not None + and actual_keys != context.recorded_keys_fp + ): + raise refused("process root-key fingerprint differs from its recorded value") + if actual_keys is None: + pending.add("processes") + return tuple(sorted(pending)), actual_keys + + +def _admitted(context: Context, value: dict[str, Any]) -> dict[str, Any]: + claim = _claim(value) + if value["cell"] != context.cell_name or value["mode"] != "enforced": + raise refused("process observation belongs to another or unenforced cell") + fields = {**cell.DEFAULTS, **value["value"]["fields"]} + for pair in (context.own_pair, context.directory_pair): + if not any(cell.matches(fields, accepted) for accepted in pair): + raise refused("process fingerprint disagrees with an accepted pair") + if value["value"]["marked"] is not True: + raise refused("process observation has no tenancy marker") + if ( + context.is_first + and claim["role"] == "server" + and value["value"]["serves"] + and value["value"].get("router") is not True + ): + raise refused("first-cell traffic process has no accepted router key") + return claim + + +def _instance_join( + value: dict[str, Any], + claim: dict[str, Any], + instances: dict[str, dict[str, Any]], +) -> bool: + instance = instances.get(value["instance_id"]) + if claim["role"] == "worker": + if instance is not None: + raise refused("worker observation also owns a web instance row") + return True + if ( + instance is None + or instance["stopped"] + or not 0 <= instance["age_ms"] <= 3 * claim["period_ms"] + ): + return False + if instance["keys_fp"] != claim["keys_fp"]: + raise refused("instance and observation root-key fingerprints disagree") + if instance["serving"] and not value["value"]["serves"]: + raise refused("nonserving observation owns a traffic instance") + return True diff --git a/server/insights/domain/cell_participants.py b/server/insights/domain/cell_participants.py new file mode 100644 index 0000000000..b11cde77c6 --- /dev/null +++ b/server/insights/domain/cell_participants.py @@ -0,0 +1,60 @@ +"""Pure participant versions and the existing cell-control refusal envelope.""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + +from server.core.cell_ch import Refusal, Refused + + +def refused(reason: str) -> Refused: + return Refused([Refusal("cell control", reason)]) + + +def latest(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + # The latest winner cannot hide a conflicting older version from another replica. + seen: dict[tuple[str, int], tuple[str, str]] = {} + for row in rows: + try: + identity = str(UUID(str(row["participant_id"]))) + version = row["version"] + state, proof = row["state"], row["host_proof"] + except (KeyError, TypeError, ValueError): + raise refused("participant census contains an unreadable row") from None + if ( + type(version) is not int + or version <= 0 + or state not in ("running", "drained", "retired") + or not isinstance(proof, str) + ): + raise refused("participant census contains an unreadable row") + previous = out.get(identity) + observed = seen.get((identity, version)) + if observed is not None and observed != (state, proof): + raise refused(f"participant {identity} has conflicting equal versions") + seen[(identity, version)] = (state, proof) + if previous is None or int(previous["version"]) < version: + out[identity] = dict(row) + return out + + +def bounded_participant(row: dict[str, Any]) -> dict[str, Any]: + """The bounded mode retains the actual UUID, birth, state and integral version.""" + from server.core.provider_process import certified + + try: + identity = str(UUID(row["participant_id"])) + proof = row["host_proof"] + if identity != row["participant_id"] or not isinstance(proof, str): + raise ValueError + if len(proof.encode()) > 2048: + raise ValueError + certified(proof) + if type(row.get("version")) is not int or not 0 < row["version"] < 2**64: + raise ValueError + except (KeyError, TypeError, ValueError, UnicodeError): + raise refused("participant census contains an unreadable birth") from None + # The existing version/state/conflict policy remains the sole selector. + return latest([row])[identity] diff --git a/server/insights/store/cell_observations.py b/server/insights/store/cell_observations.py new file mode 100644 index 0000000000..2c1c837f21 --- /dev/null +++ b/server/insights/store/cell_observations.py @@ -0,0 +1,461 @@ +"""Complete bounded process census for the explicit later-attach component.""" + +from __future__ import annotations + +import asyncio +import math +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any + +from server.core import cell, ch_names +from server.core.cell_ch import WHOLE, Topology, complete, per_host, quote +from server.core.ch_bounded import BoundedReadError, decode_json, read_rows +from server.insights.domain import cell_observations as domain +from server.insights.domain.cell_participants import refused +from server.insights.store import provider_participants + +TAG = "cell.attach.census" +_TABLES = ("_provider_participants", "cell_heartbeats", "instances") +_COLUMNS = { + "_provider_participants": { + "participant_id": "UUID", + "state": "LowCardinality(String)", + "version": "UInt64", + "host_proof": "String", + }, + "cell_heartbeats": { + "instance_id": "String", + "cell": "String", + "tenancy_mode": "String", + "fingerprint": "String", + "heartbeat_at": "DateTime64(3)", + }, + "instances": { + "instance_id": "String", + "keys_fp": "String", + "serving": "UInt8", + "stopped": "UInt8", + "heartbeat_at": "DateTime64(3)", + }, +} + + +class _Reads: + """One complete census shares one raw allowance across all reads and passes.""" + + def __init__(self, ch: Any, attempt: domain.Attempt, deadline: float) -> None: + remaining = deadline - time.monotonic() + if not math.isfinite(deadline) or not 0 < remaining <= 7200: + raise refused("census has no finite remaining workflow allowance") + if attempt.deadline is not None and deadline != attempt.deadline: + raise refused("census cannot reset its workflow deadline") + attempt.deadline = deadline + self.ch, self.deadline = ch, deadline + self.raw_bytes_used = 0 + + def left(self) -> float: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise refused("census deadline exhausted") + return remaining + + @asynccontextmanager + async def raw_stream( + self, sql: str, params: Any = None, *, settings: Any = None + ) -> AsyncIterator[AsyncIterator[bytes]]: + effective = { + **WHOLE, + "skip_unavailable_shards": 0, + "prefer_localhost_replica": 1, + "default_format": "JSONEachRow", + **(settings or {}), + } + remaining = self.left() + for name in ("max_execution_time", "receive_timeout", "distributed_ddl_task_timeout"): + if name in effective: + effective[name] = min(float(effective[name]), remaining) + async with asyncio.timeout(remaining): + async with self.ch.raw_stream(sql, params=params or {}, settings=effective) as chunks: + + async def limited() -> AsyncIterator[bytes]: + async for chunk in chunks: + if self.raw_bytes_used + len(chunk) > domain.METADATA_BYTES: + raise BoundedReadError("Aggregate census raw byte limit exceeded") + self.raw_bytes_used += len(chunk) + yield chunk + + yield limited() + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + values = params or {} + # A JSONEachRow object needs at least three bytes: {} followed by newline. + max_rows = domain.METADATA_BYTES // 3 + if "replica_limit" in values: + max_rows = domain.REPLICAS + 1 + if "LIMIT 1025 BY h" in sql: + max_rows = (domain.PARTICIPANTS + 1) * domain.REPLICAS + result = await read_rows( + sql, + values, + ch=self, + deadline=self.deadline, + row_bytes=domain.FINGERPRINT_BYTES + 32 * domain.FIELD_BYTES, + total_bytes=domain.METADATA_BYTES - self.raw_bytes_used, + max_rows=max_rows, + tag=TAG, + ) + self.left() + _metadata_fields(result) + if "groupArrayIf(host_name, is_local)" in sql: + _host_names(result) + return result + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any] | None: + rows = await self.query_all(sql, params) + if len(rows) > 1: + raise refused("single-row census query returned more than one row") + return rows[0] if rows else None + + async def raw_bytes(self, sql: str, params: Any = None, *, settings: Any = None) -> bytes: + chunks = [] + async with self.raw_stream(sql, params, settings=settings) as stream: + async for chunk in stream: + chunks.append(chunk) + body = b"".join(chunks) + lines = body.splitlines() + if len(lines) > domain.REPLICAS: + raise refused("replica synchronization status limit exceeded") + for line in lines: + if len(line) > domain.FINGERPRINT_BYTES: + raise refused("replica synchronization status row exceeds its bound") + value = decode_json(line.decode("utf-8")) + if not isinstance(value, dict): + raise refused("unreadable replica synchronization status") + _metadata_fields([value]) + return body + + async def execute(self, sql: str, params: Any = None, *, settings: Any = None) -> None: + remaining = self.left() + bounds = dict(settings or {}) + for name in ("max_execution_time", "receive_timeout"): + bounds[name] = min(float(bounds.get(name, remaining)), remaining) + async with asyncio.timeout(remaining): + await self.ch.execute(sql, params, settings=bounds) + + +def _metadata_fields(rows: list[dict[str, Any]]) -> None: + # Fingerprint has its own larger bound and strict nested decoder before retention. + for row in rows: + for key, value in row.items(): + domain.text(key) + if key != "fingerprint": + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + domain.text(item, empty=True) + elif isinstance(item, list): + pending.extend(item) + elif isinstance(item, dict): + for name, child in item.items(): + domain.text(name) + pending.append(child) + + +def _host_names(rows: list[dict[str, Any]]) -> None: + if len(rows) > domain.REPLICAS: + raise refused("answering replica limit exceeded") + for row in rows: + domain.text(row.get("h")) + names = row.get("names") + if not isinstance(names, list) or len(names) != 1: + raise refused("answering replica has no unique configured name") + domain.text(names[0]) + + +async def _catalog(ch: _Reads, topology: Topology, names: dict[str, str]) -> None: + params = {"db": ch_names.database(), "tables": list(_TABLES)} + tables = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, name, engine, engine_full FROM {source:tables} " + "WHERE database={db:String} AND name IN {tables:Array(String)}", + params, + ) + columns = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, table, name, type FROM {source:columns} " + "WHERE database={db:String} AND table IN {tables:Array(String)}", + params, + ) + reference = None + for host, found in tables.items(): + if len(found) != len(_TABLES) or len({r.get("name") for r in found}) != len(_TABLES): + raise refused("census table catalog is missing or duplicated") + kinds = {domain.text(r["name"]): domain.text(r["engine"]) for r in found} + if kinds.get("cell_heartbeats") != "KeeperMap" or kinds.get("instances") != "KeeperMap": + raise refused("observation or instance table is not KeeperMap") + typed = tuple( + sorted( + (domain.text(r["table"]), domain.text(r["name"]), domain.text(r["type"])) + for r in columns.get(host, []) + ) + ) + if len({(t, n) for t, n, _ in typed}) != len(typed): + raise refused("census column catalog is duplicated") + for table, expected in _COLUMNS.items(): + actual = {n: kind for t, n, kind in typed if t == table} + if any(actual.get(name) != kind for name, kind in expected.items()): + raise refused("census table has an unsupported column type") + keeper = tuple( + sorted( + (r["name"], domain.text(r["engine_full"])) + for r in found + if r["name"] != "_provider_participants" + ) + ) + view = (tuple(sorted(kinds.items())), typed, keeper) + if reference is not None and reference != view: + raise refused("census replicas disagree on their table catalog") + reference = view + + +def _source(table: str, topology: Topology) -> tuple[str, dict[str, Any]]: + source = f"{quote(ch_names.database())}.`{table}`" + if topology.multi: + return f"clusterAllReplicas({{cluster:String}}, {source})", {"cluster": topology.cluster} + return source, {} + + +async def _observations(ch: _Reads, topology: Topology) -> list[dict[str, Any]]: + source, params = _source("cell_heartbeats", topology) + sql = ( + "SELECT hostName() AS h, " + "if(length(census_observation.instance_id)<=2048, " + "census_observation.instance_id, '') AS instance_id, " + "if(length(census_observation.cell)<=2048, census_observation.cell, '') AS " + "cell, length(census_observation.cell) AS cell_bytes, " + "if(length(census_observation.tenancy_mode)<=2048, " + "census_observation.tenancy_mode, '') AS tenancy_mode, " + "length(census_observation.tenancy_mode) AS tenancy_mode_bytes, " + "if(length(census_observation.fingerprint)<=262144, " + "census_observation.fingerprint, '') AS fingerprint, " + "length(census_observation.fingerprint) AS fingerprint_bytes, " + "toUnixTimestamp64Milli(census_observation.heartbeat_at) AS generation, " + "dateDiff('millisecond', census_observation.heartbeat_at, now64(3)) AS age_ms " + f"FROM {source} AS census_observation LIMIT 1025 BY h" + ) + return await ch.query_all(complete(sql, sequential=1), params) + + +async def _instances(ch: _Reads, topology: Topology) -> list[dict[str, Any]]: + source, params = _source("instances", topology) + sql = ( + "SELECT hostName() AS h, " + "if(length(census_instance.instance_id)<=2048, census_instance.instance_id, " + "'') AS instance_id, " + "if(length(census_instance.keys_fp)<=2048, census_instance.keys_fp, '') AS keys_fp, " + "length(census_instance.keys_fp) AS keys_fp_bytes, serving, stopped, " + "toUnixTimestamp64Milli(census_instance.heartbeat_at) AS generation, " + "dateDiff('millisecond', census_instance.heartbeat_at, now64(3)) AS age_ms " + f"FROM {source} AS census_instance LIMIT 1025 BY h" + ) + return await ch.query_all(complete(sql, sequential=1), params) + + +def _projected_field(row: dict[str, Any], name: str) -> None: + value = domain.text(row.get(name), empty=True) + size = row.get(name + "_bytes") + if type(size) is not int or not 0 <= size <= domain.FIELD_BYTES or len(value.encode()) != size: + raise refused("process field exceeds or disagrees with its projection bound") + + +def _replicas(rows: list[dict[str, Any]], topology: Topology, names: dict[str, str]) -> None: + counts: dict[str, int] = {} + identities: set[str] = set() + for row in rows: + host = domain.text(row.get("h")) + if topology.multi and host not in names: + raise refused("process row came from an unconfigured replica") + if host not in counts and len(counts) >= (len(names) if topology.multi else 1): + raise refused("process row came from an extra replica") + count = counts.get(host, 0) + 1 + if count > 1024: + raise refused("process replica row limit exceeded") + counts[host] = count + identity = domain.identity(row.get("instance_id"), instance=True) + if identity not in identities and len(identities) >= 1024: + raise refused("logical observation identity limit exceeded") + identities.add(identity) + + +def _collapse_observations( + rows: list[dict[str, Any]], attempt: domain.Attempt +) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + replicas: set[tuple[str, str]] = set() + for row in rows: + _projected_field(row, "cell") + _projected_field(row, "tenancy_mode") + value = observation(row) + key = value["instance_id"] + replica = (row["h"], key) + if replica in replicas: + raise refused("one replica returned a duplicated observation") + replicas.add(replica) + previous = out.get(key) + if previous: + stable = {k: v for k, v in value.items() if k not in {"age_ms", "generation"}} + old = {k: v for k, v in previous.items() if k not in {"age_ms", "generation"}} + if stable != old: + raise refused("replicas disagree on the process observation") + # Every replica must be fresh; keep the worst observed age for the judge. + value["age_ms"] = ( + -1 + if min(value["age_ms"], previous["age_ms"]) < 0 + else max(value["age_ms"], previous["age_ms"]) + ) + value["generation"] = max(value["generation"], previous["generation"]) + attempt.retain_observation(value) + out[key] = value + return out + + +def _collapse_instances(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + replicas: set[tuple[str, str]] = set() + for row in rows: + key = domain.identity(row.get("instance_id"), instance=True) + _projected_field(row, "keys_fp") + generation = row.get("generation") + value = { + "keys_fp": domain.text(row.get("keys_fp"), empty=True), + **{name: row.get(name) for name in ("serving", "stopped", "age_ms", "generation")}, + } + if ( + any( + type(value[name]) is not int + for name in ("serving", "stopped", "age_ms", "generation") + ) + or value["serving"] not in (0, 1) + or value["stopped"] not in (0, 1) + or type(generation) is not int + or generation < 0 + ): + raise refused("unreadable web instance metadata") + replica = (row["h"], key) + if replica in replicas: + raise refused("one replica returned a duplicated web instance") + replicas.add(replica) + previous = out.get(key) + if previous: + if any(previous[name] != value[name] for name in ("keys_fp", "serving", "stopped")): + raise refused("replicas disagree on a web instance") + value["age_ms"] = ( + -1 + if min(value["age_ms"], previous["age_ms"]) < 0 + else max(value["age_ms"], previous["age_ms"]) + ) + value["generation"] = max(value["generation"], previous["generation"]) + out[key] = value + return out + + +@dataclass(frozen=True) +class Snapshot: + """Complete selected evidence with explicit unresolved obligations; no activation.""" + + participants: dict[str, dict[str, Any]] + observations: dict[str, dict[str, Any]] + pending: tuple[str, ...] + keys_fp: str | None + raw_bytes: int + retained_bytes: int + + +async def read_cell( + ch: Any, attempt: domain.Attempt, context: domain.Context, *, deadline: float +) -> Snapshot: + """Read participants, all observations/instances, then participants again.""" + context.validate() + if attempt.cell_name is not None and attempt.cell_name != context.cell_name: + raise refused("census attempt cannot change its cell") + attempt.cell_name = context.cell_name + reader = _Reads(ch, attempt, deadline) + first = await provider_participants.census(reader, bounded=True) + attempt.capture(first.rows) + await _catalog(reader, first.topology, first.names) + raw_observations = await _observations(reader, first.topology) + _replicas(raw_observations, first.topology, first.names) + raw_instances = await _instances(reader, first.topology) + _replicas(raw_instances, first.topology, first.names) + second = await provider_participants.census(reader, bounded=True) + if (first.topology, first.names, first.engine) != ( + second.topology, + second.names, + second.engine, + ): + raise refused("participant topology changed during the observation join") + attempt.capture(second.rows) + observed = _collapse_observations(raw_observations, attempt) + instances = _collapse_instances(raw_instances) + pending, keys_fp = attempt.judge(context, second.rows, observed, instances) + reader.left() + return Snapshot( + second.rows, observed, pending, keys_fp, reader.raw_bytes_used, attempt.retained_bytes + ) + + +def _finite_fields(value: dict[str, Any]) -> None: + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, str): + domain.text(item, empty=True) + elif isinstance(item, dict): + for key, child in item.items(): + domain.text(key) + pending.append(child) + elif isinstance(item, list): + pending.extend(item) + + +def observation(row: dict[str, Any]) -> dict[str, Any]: + """Decode only the bounded projection; CH evaluates age at the actual read.""" + instance = domain.identity(row.get("instance_id"), instance=True) + raw = row.get("fingerprint") + if not isinstance(raw, str) or type(row.get("fingerprint_bytes")) is not int: + raise refused("unreadable observation fingerprint") + if ( + row["fingerprint_bytes"] > domain.FINGERPRINT_BYTES + or len(raw.encode()) != row["fingerprint_bytes"] + ): + raise refused("observation fingerprint exceeds its bound") + value = decode_json(raw) + if not isinstance(value, dict): + raise refused("unreadable observation fingerprint") + fp = {key: value[key] for key in ("v", "fields") if key in value} + if not cell.readable_fingerprint(fp): + raise refused("unreadable observation fingerprint") + # Fingerprint fields use the strict codec and their aggregate byte allowance. + _finite_fields({key: item for key, item in value.items() if key != "fields"}) + participant = domain.identity(value.get("participant_id")) + if type(row.get("generation")) is not int or row["generation"] < 0: + raise refused("unreadable observation generation") + if type(row.get("age_ms")) is not int: + raise refused("unreadable observation time") + return { + "instance_id": instance, + "participant_id": participant, + "cell": domain.text(row.get("cell"), empty=True), + "mode": domain.text(row.get("tenancy_mode"), empty=True), + "generation": row["generation"], + "age_ms": row["age_ms"], + "value": value, + } diff --git a/server/insights/store/provider_participants.py b/server/insights/store/provider_participants.py index e03c5b22bf..7e658b7a81 100644 --- a/server/insights/store/provider_participants.py +++ b/server/insights/store/provider_participants.py @@ -1,9 +1,21 @@ """Full-row process registrations, heartbeats and explicit transition acknowledgements.""" +from dataclasses import dataclass from time import time_ns -from typing import Any +from typing import Any, Literal, overload from uuid import UUID +from server.core import cell_sync, ch_names +from server.core.cell_ch import ( + TRANSACTION_S, + WHOLE_SQL, + Refused, + Topology, + complete, + per_host, + quote, + read_topology, +) from server.core.ch import sequential_reads from server.core.ch_errors import error_code from server.core.provider_process import ( @@ -14,6 +26,8 @@ ) from server.core.repo_base import _CHClient, _default_client from server.core.retirement import RetirementProof, get_gate +from server.insights.domain.cell_participants import bounded_participant, latest, refused +from server.insights.store import cell_catalog from server.settings import settings TABLE = "loom._provider_participants" @@ -226,3 +240,181 @@ def participant_sync(ch: Any, identity: UUID | str) -> dict[str, Any] | None: "version": int(version), "expired": bool(expired), } + + +@dataclass(frozen=True) +class ParticipantCensus: + rows: dict[str, dict[str, Any]] + topology: Topology + names: dict[str, str] + engine: str + + +@overload +async def census(ch: Any, *, bounded: Literal[False] = False) -> dict[str, dict[str, Any]]: ... + + +@overload +async def census(ch: Any, *, bounded: Literal[True]) -> ParticipantCensus: ... + + +async def census( + ch: Any, *, bounded: bool = False +) -> dict[str, dict[str, Any]] | ParticipantCensus: + """Synchronize after the durable cutoff; every configured replica must answer.""" + topology = await read_topology(ch, max_replicas=64) if bounded else await read_topology(ch) + if topology.cluster and not topology.entries: + raise refused("participant census cannot resolve the configured cluster") + if problems := topology.refusals(): + raise Refused(problems) + names = await cell_catalog.host_names(ch, topology) if topology.multi else {} + if bounded and topology.cluster and not topology.multi: + names = await bounded_local_replica(ch, topology) + if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: + raise Refused(problems) + database = ch_names.database() + tables = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, engine FROM {source:tables} WHERE database={db:String} AND " + "name='_provider_participants'" + ), + {"db": database}, + ) + if not tables or any(len(rows) != 1 for rows in tables.values()): + raise refused("participant table is missing from a configured replica") + engines = {str(rows[0]["engine"]) for rows in tables.values()} + if len(engines) != 1 or not next(iter(engines)).endswith("MergeTree"): + raise refused("participant replicas have unreadable or different table engines") + engine = next(iter(engines)) + if bounded and engine not in ( + "ReplacingMergeTree", + "ReplicatedReplacingMergeTree", + "SharedReplacingMergeTree", + ): + raise refused("participant table has an unsupported replacement policy") + if engine.startswith("Replicated"): + replica_rows = await per_host( + ch, + topology, + names, + ( + "SELECT hostName() AS h, database, table, zookeeper_path, replica_name, " + "replica_is_active FROM {source:replicas} WHERE database={db:String} AND " + "table='_provider_participants'" + ), + {"db": database}, + ) + if any(len(value) != 1 for value in replica_rows.values()) or cell_catalog.extra_replicas( + row for value in replica_rows.values() for row in value + ): + raise refused("participant replica set includes an unverified replica") + if bounded: + bounded_replica_paths(replica_rows) + budget = cell_sync.Budget(TRANSACTION_S) + await cell_sync._system( + ch, + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " + f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", + topology, + budget, + "participant census", + ) + source = f"{quote(database)}.`_provider_participants`" + params = {} + if topology.multi: + source = f"clusterAllReplicas({{cluster:String}}, {source})" + params["cluster"] = topology.cluster + sql = f"SELECT participant_id, state, version, host_proof FROM {source} FINAL" + if bounded: + sql = ( + f"SELECT hostName() AS h, participant_id, state, version, " + "if(length(census_participant.host_proof) <= 2048, " + "census_participant.host_proof, '') AS host_proof, " + f"length(census_participant.host_proof) AS proof_bytes " + f"FROM {source} AS census_participant LIMIT 1025 BY h" + ) + rows = await ch.query_all(complete(sql, sequential=int(engine.startswith("Shared"))), params) + if bounded: + rows = bounded_participants(rows, topology, names) + return ParticipantCensus(latest(rows), topology, names, engine) + return latest(rows) + + +def bounded_participants( + rows: list[dict[str, Any]], topology: Any, names: Any +) -> list[dict[str, Any]]: + """Validate replica duplication and cap+1 before the shared latest selector.""" + counts: dict[str, int] = {} + logical: set[str] = set() + out = [] + for row in rows: + host = row.get("h") + if not isinstance(host, str) or not host or len(host.encode()) > 2048: + raise refused("participant row has an unreadable replica") + if topology.multi and host not in names: + raise refused("participant row came from an unconfigured replica") + if host not in counts and len(counts) >= (len(names) if topology.multi else 1): + raise refused("participant row came from an extra replica") + count = counts.get(host, 0) + 1 + if count > 1024: + raise refused("participant replica row limit exceeded") + counts[host] = count + if type(row.get("proof_bytes")) is not int or not 0 <= row["proof_bytes"] <= 2048: + raise refused("participant proof exceeds its field bound") + if ( + not isinstance(row.get("host_proof"), str) + or len(row["host_proof"].encode()) != row["proof_bytes"] + ): + raise refused("participant proof length disagrees with its projection") + value = bounded_participant( + {key: row[key] for key in ("participant_id", "state", "version", "host_proof")} + ) + identity = value["participant_id"] + if identity not in logical and len(logical) >= 1024: + raise refused("logical participant limit exceeded") + logical.add(identity) + out.append(value) + return out + + +def bounded_replica_paths(rows: dict[str, list[dict[str, Any]]]) -> None: + """All replicas synchronize one table identity, not unrelated same-named tables.""" + paths: set[str] = set() + names: set[str] = set() + for values in rows.values(): + row = values[0] + path, name = row.get("zookeeper_path"), row.get("replica_name") + if ( + not isinstance(path, str) + or not path + or len(path.encode()) > 2048 + or not isinstance(name, str) + or not name + or len(name.encode()) > 2048 + or name in names + ): + raise refused("participant replica identity is unreadable or duplicated") + paths.add(path) + names.add(name) + if len(paths) != 1: + raise refused("participant replicas do not share one table identity") + for values in rows.values(): + listed = values[0].get("replica_is_active") + if ( + not isinstance(listed, dict) + or set(listed) != names + or any(type(v) is not int or v != 1 for v in listed.values()) + ): + raise refused("participant replica membership is missing, inactive or unreadable") + + +async def bounded_local_replica(ch: Any, topology: Topology) -> dict[str, str]: + """A configured one-host cluster must describe the server read locally.""" + names = await cell_catalog.host_names(ch, topology) + local = await ch.query_one("SELECT hostName() AS h " + WHOLE_SQL) + if local is None or local.get("h") not in names: + raise refused("local participant reader is outside the configured one-host cluster") + return names diff --git a/server/router/app.py b/server/router/app.py index 87cc8d775e..4909335343 100644 --- a/server/router/app.py +++ b/server/router/app.py @@ -67,7 +67,7 @@ from server.core import cell as cell_state from server.core.cell_probe import PATH as PROBE_PATH -from server.router import directory, fingerprints, routing +from server.router import directory, routing from server.router.limits import ( ATTACH_BATCH, ATTACH_BEGIN_MAX_BYTES, @@ -831,7 +831,9 @@ async def configuration(self, request: Request) -> JSONResponse: if set(body) != {"action", "target_fp", "keys_fp", "nonces"}: return _refused(400, "configuration requires action, target_fp, keys_fp and nonces") action, target, keys_fp = body["action"], body["target_fp"], body["keys_fp"] - if action not in ("config", "finish", "revert") or not fingerprints.readable(target): + if action not in ("config", "finish", "revert") or not cell_state.readable_fingerprint( + target + ): return _refused(400, "configuration requires a supported action and readable target_fp") if _text(body, "keys_fp", needed=True) is None: return _refused(400, f"keys_fp is text of 1 to {_TEXT_MAX} characters, no NUL") diff --git a/server/router/directory.py b/server/router/directory.py index aa7ac03328..a5781aa0b0 100644 --- a/server/router/directory.py +++ b/server/router/directory.py @@ -1159,7 +1159,7 @@ async def read(conn: Any, found: Attaching) -> list[str]: accepted = [ value for value in (found.row.current_fp, found.row.next_fp) if value is not None ] - if not accepted or any(not fingerprints.readable(value) for value in accepted): + if not accepted or any(not cell_state.readable_fingerprint(value) for value in accepted): raise OperatorRefused( "configured placement proof needs readable directory reservations" ) diff --git a/server/router/fingerprints.py b/server/router/fingerprints.py index 0a818f5a3b..af3ccd0035 100644 --- a/server/router/fingerprints.py +++ b/server/router/fingerprints.py @@ -10,55 +10,9 @@ from server.router.limits import MAX_BODY_BYTES -def readable(value: Any) -> bool: - """Known fields have exact types; unknown JSON fields remain reserved and intact.""" - if not isinstance(value, dict) or set(value) != {"v", "fields"}: - return False - if type(value["v"]) is not int or value["v"] != cell.FINGERPRINT_VERSION: - return False - fields = value["fields"] - if not isinstance(fields, dict): - return False - try: - # Reject cycles, non-JSON values and nonfinite floats before walking nested fields. - json.dumps(value, allow_nan=False) - except (TypeError, ValueError, RecursionError): - return False - pending = [value] - while pending: - item = pending.pop() - if isinstance(item, str): - try: - if "\0" in item: - return False - item.encode() - except UnicodeError: - return False - elif isinstance(item, dict): - if any(not isinstance(key, str) for key in item): - return False - pending.extend(item.keys()) - pending.extend(item.values()) - elif isinstance(item, list): - pending.extend(item) - elif item is not None and type(item) not in (bool, int, float): - return False - for name, default in cell.DEFAULTS.items(): - item = fields.get(name, default) - if type(item) is not type(default): - return False - if isinstance(item, list) and any(not isinstance(part, str) for part in item): - return False - for name in ("configured_orgs", "end_user_orgs"): - for org in fields.get(name, []): - if cell.named_org(org) is None or cell.org_name_refusal(org): - return False - return True - - def registration_refusal(value: Any, registered: Mapping[str, Any], name: str = "candidate") -> str: """A fingerprint accepts only the registered stores and host.""" - if not readable(value): + if not cell.readable_fingerprint(value): return f"{name}: unreadable fingerprint; change fingerprints with `loom cell update`" expected = { "deployment_id": registered["deployment_id"], @@ -121,7 +75,7 @@ def reservation_refusal(accepted: Sequence[dict[str, Any]], other: Sequence[dict Registration identity, placement winners and transaction locks are the caller's checks. Empty `other` means registration has not reserved a fingerprint yet. """ - if not accepted or any(not readable(value) for value in (*accepted, *other)): + if not accepted or any(not cell.readable_fingerprint(value) for value in (*accepted, *other)): return "unreadable fingerprint reservation" for a in accepted: for b in other: @@ -145,8 +99,8 @@ def configuration_pair( action: str, ) -> tuple[dict[str, Any], dict[str, Any] | None]: """Select the reserved pair without guessing the direction of an interrupted update.""" - if any(not readable(value) for value in (current, target)) or ( - pending is not None and not readable(pending) + if any(not cell.readable_fingerprint(value) for value in (current, target)) or ( + pending is not None and not cell.readable_fingerprint(pending) ): raise ConfigurationRefused("configuration update needs readable fingerprints") assert current is not None diff --git a/tests/integration/test_cell_observation_census.py b/tests/integration/test_cell_observation_census.py new file mode 100644 index 0000000000..4d0af9aa86 --- /dev/null +++ b/tests/integration/test_cell_observation_census.py @@ -0,0 +1,244 @@ +"""The bounded reader evaluates owned KeeperMap rows through real ClickHouse I/O.""" + +from __future__ import annotations + +import contextlib +import copy +import json +import os +import time +from datetime import timedelta +from uuid import UUID, uuid4 + +import pytest + +from server.core import cell +from server.core.cell_ch import Refused +from server.core.ch import CHClient, query_tag +from server.insights.domain.cell_observations import Attempt, Context +from server.insights.store.cell_observations import read_cell +from server.settings import settings + +KEYS = "a" * 64 +PROOF = "host|boot|container|namespace|123|456|||cell-retirement-v1" +SERVER = UUID(int=1) +WORKER = UUID(int=2) +_COLUMNS = { + "_provider_participants": ( + "participant_id UUID, state LowCardinality(String), version UInt64, host_proof String" + ), + "cell_heartbeats": ( + "instance_id String, cell String DEFAULT '', tenancy_mode String DEFAULT '', " + "fingerprint String DEFAULT '', heartbeat_at DateTime64(3) DEFAULT now64(3)" + ), + "instances": ( + "instance_id String, deployment UInt8 DEFAULT 0, label String DEFAULT '', " + "host String DEFAULT '', pid UInt32 DEFAULT 0, build_commit String DEFAULT '', " + "schema_version UInt32 DEFAULT 0, serving UInt8 DEFAULT 1, workers_lease String DEFAULT '', " + "settings_fp String DEFAULT '', keys_fp String DEFAULT '', config_fp String DEFAULT '', " + "started_at DateTime64(3) DEFAULT now64(3), " + "heartbeat_at DateTime64(3) DEFAULT now64(3), stopped UInt8 DEFAULT 0" + ), +} + + +@pytest.fixture +async def census_store(monkeypatch): + if not os.environ.get("LOOM_CH_HOST"): + pytest.skip("Set LOOM_CH_HOST to the owned disposable ClickHouse") + database = "census_" + uuid4().hex + client = CHClient() + cleanup = contextlib.AsyncExitStack() + + async def drop_owned(): + await client.execute("DROP DATABASE {db:Identifier} SYNC", {"db": database}) + assert await client.query_one( + "SELECT count() AS n FROM system.tables WHERE database={db:String}", + {"db": database}, + ) == {"n": 0} + assert await client.query_one( + "SELECT count() AS n FROM system.databases WHERE name={db:String}", + {"db": database}, + ) == {"n": 0} + + with query_tag("test.cell.observation_census"): + try: + cleanup.push_async_callback(client.disconnect) + await client.connect() + await client.execute("CREATE DATABASE {db:Identifier}", {"db": database}) + cleanup.push_async_callback(drop_owned) + for table, columns in _COLUMNS.items(): + params = {"db": database, "table": table} + if table == "_provider_participants": + engine = "ReplacingMergeTree(version) ORDER BY participant_id" + else: + engine = "KeeperMap({root:String}) PRIMARY KEY instance_id" + params["root"] = f"/loom_it_census/{database}/{table}" + await client.execute( + f"CREATE TABLE {{db:Identifier}}.{{table:Identifier}} ({columns}) " + f"ENGINE = {engine}", + params, + ) + assert await client.query_one( + "SELECT count() AS n FROM {db:Identifier}.{table:Identifier}", params + ) == {"n": 0} + catalog = await client.query_all( + "SELECT name, engine, engine_full FROM system.tables " + "WHERE database={db:String} ORDER BY name", + {"db": database}, + ) + assert {row["name"] for row in catalog} == set(_COLUMNS) + for row in catalog: + if row["name"] == "_provider_participants": + assert row["engine"] == "ReplacingMergeTree" + else: + assert row["engine"] == "KeeperMap" + assert f"/loom_it_census/{database}/{row['name']}" in row["engine_full"] + monkeypatch.setattr(settings, "ch_database", database) + monkeypatch.setattr(settings, "ch_cluster", "") + yield client + except BaseException as first: # Keep setup/body failure through owned cleanup. + try: + await cleanup.aclose() + except BaseException as error: # noqa: BLE001 — the first failure remains primary. + first.add_note(f"Owned census cleanup also failed: {type(error).__name__}") + raise + else: + await cleanup.aclose() + + +def context(): + fingerprint = cell.as_value({"deployment_id": "dep", "oidc_client_id": "x" * 3000}) + assert cell.readable_fingerprint(fingerprint) + return Context("later", False, (fingerprint,), (fingerprint,), KEYS) + + +def participant(identity, *, state="running", version=1): + return { + "participant_id": identity, + "state": state, + "version": version, + "host_proof": PROOF, + } + + +def observation(identity, role, configured, *, extra=None): + value = copy.deepcopy(configured.own_pair[0]) + value.update( + participant_id=str(identity), + marked=True, + serves=role == "server", + router=False, + attach_observation={ + "protocol": 1, + "role": role, + "participant_id": str(identity), + "host_proof": PROOF, + "keys_fp": KEYS, + "accepting": True, + "period_ms": 60000, + }, + ) + value.update(extra or {}) + return { + "instance_id": identity.hex, + "cell": configured.cell_name, + "tenancy_mode": "enforced", + "fingerprint": json.dumps(value), + } + + +async def seed(client, configured, *, worker_observation=True): + await client.insert("loom._provider_participants", [participant(SERVER), participant(WORKER)]) + rows = [observation(SERVER, "server", configured)] + if worker_observation: + rows.append(observation(WORKER, "worker", configured)) + await client.insert("loom.cell_heartbeats", rows) + await client.insert("loom.instances", [{"instance_id": SERVER.hex, "keys_fp": KEYS}]) + + +async def test_native_reader_preserves_large_fields_and_refuses_other_oversized_metadata( + census_store, +): + client, configured, attempt = census_store, context(), Attempt() + await seed(client, configured) + deadline = time.monotonic() + 30 + snapshot = await read_cell(client, attempt, configured, deadline=deadline) + assert snapshot.pending == () and snapshot.keys_fp == KEYS + assert set(snapshot.participants) == {str(SERVER), str(WORKER)} + assert { + value["value"]["attach_observation"]["role"] for value in snapshot.observations.values() + } == {"server", "worker"} + assert 0 < snapshot.raw_bytes <= 8 * 1024**2 + assert 0 < snapshot.retained_bytes <= 8 * 1024**2 + for value in snapshot.observations.values(): + assert value["value"]["fields"]["oidc_client_id"] == "x" * 3000 + assert type(value["generation"]) is int and value["generation"] > 0 + assert type(value["age_ms"]) is int and 0 <= value["age_ms"] <= 180000 + await client.insert( + "loom.cell_heartbeats", + [observation(WORKER, "worker", configured, extra={"metadata_note": "x" * 2049})], + ) + with pytest.raises(Refused, match="process metadata exceeds its field bound"): + await read_cell(client, attempt, configured, deadline=deadline) + + +async def test_native_retirement_closes_only_the_missing_worker_in_the_same_attempt(census_store): + client, configured, attempt = census_store, context(), Attempt() + await seed(client, configured, worker_observation=False) + deadline = time.monotonic() + 30 + before = await read_cell(client, attempt, configured, deadline=deadline) + assert before.pending == (str(WORKER),) + first = copy.deepcopy(attempt.participants[str(WORKER)]["first"]) + birth = attempt.participants[str(WORKER)]["birth"] + await client.insert( + "loom._provider_participants", [participant(WORKER, state="retired", version=2)] + ) + after = await read_cell(client, attempt, configured, deadline=deadline) + assert after.pending == () and after.keys_fp == KEYS + assert attempt.participants[str(WORKER)]["closed"] is True + assert attempt.participants[str(WORKER)]["first"] == first + assert attempt.participants[str(WORKER)]["birth"] == birth + assert len(attempt.participants) == 2 and 0 < attempt.retained_bytes <= 8 * 1024**2 + assert attempt.deadline == deadline + + +@pytest.mark.parametrize("minutes", [-10, 10]) +async def test_native_server_clock_marks_stale_or_future_observations_unresolved( + census_store, minutes +): + client, configured = census_store, context() + await seed(client, configured) + clock = await client.query_one("SELECT now64(3) AS stamp") + assert clock is not None + row = observation(WORKER, "worker", configured) + row["heartbeat_at"] = clock["stamp"] + timedelta(minutes=minutes) + await client.insert("loom.cell_heartbeats", [row]) + snapshot = await read_cell(client, Attempt(), configured, deadline=time.monotonic() + 30) + assert snapshot.pending == (str(WORKER),) + age = snapshot.observations[WORKER.hex]["age_ms"] + assert age > 180000 if minutes < 0 else age < 0 + + +async def test_native_identity_allowance_retains_closed_history_across_reads(census_store): + client, configured, attempt = census_store, context(), Attempt() + await client.insert( + "loom._provider_participants", + [participant(SERVER)] + [participant(UUID(int=n), state="retired") for n in range(2, 1025)], + ) + await client.insert("loom.cell_heartbeats", [observation(SERVER, "server", configured)]) + await client.insert("loom.instances", [{"instance_id": SERVER.hex, "keys_fp": KEYS}]) + deadline = time.monotonic() + 30 + snapshot = await read_cell(client, attempt, configured, deadline=deadline) + assert snapshot.pending == () and len(attempt.participants) == 1024 + retained = copy.deepcopy(attempt.participants) + retained_bytes = attempt.retained_bytes + await client.execute("TRUNCATE TABLE loom._provider_participants") + await client.insert( + "loom._provider_participants", + [participant(SERVER), participant(UUID(int=1025), state="retired")], + ) + with pytest.raises(Refused, match="cumulative participant identity limit exceeded"): + await read_cell(client, attempt, configured, deadline=deadline) + assert attempt.participants == retained and attempt.retained_bytes == retained_bytes + assert attempt.deadline == deadline diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index b80384b90e..aaed352e13 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -365,7 +365,7 @@ def function(path: str, name: str) -> ast.AST: "_prepare_locked", [ "cell_control.claim(conn, call, snapshot)", - "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=cell_control.TRANSACTION_S)", + "owner.call(lambda: _write_rows(conn, rows, answer, current, nxt), total_s=TRANSACTION_S)", ], ), ( diff --git a/tests/unit/test_cell_fingerprint_reservations.py b/tests/unit/test_cell_fingerprint_reservations.py index 461eadd657..bcdcec3e15 100644 --- a/tests/unit/test_cell_fingerprint_reservations.py +++ b/tests/unit/test_cell_fingerprint_reservations.py @@ -130,7 +130,7 @@ def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: import httpx - from server.router.fingerprints import placement_refusal, readable + from server.router.fingerprints import placement_refusal from server.router.limits import MAX_BODY_BYTES deployment = "11111111-1111-4111-8111-111111111111" @@ -139,7 +139,7 @@ def test_live_size_cannot_admit_an_oversized_operator_placement() -> None: room = MAX_BODY_BYTES - len(json.dumps(live)) - 6 * cell.ORG_MAX_BYTES value["fields"]["oidc_client_id"] = "x" * room org = "\x01" * cell.ORG_MAX_BYTES - assert cell.named_org(org) == org and readable(value) + assert cell.named_org(org) == org and cell.readable_fingerprint(value) live["org"] = org operator = {"org": org, "deployment_id": deployment, "fingerprint": value} assert ( @@ -162,7 +162,7 @@ def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deploymen import httpx - from server.router.fingerprints import placement_refusal, readable + from server.router.fingerprints import placement_refusal from server.router.limits import MAX_BODY_BYTES value = _value(deployment_id=deployment, oidc_client_id="") @@ -173,7 +173,11 @@ def test_both_placement_envelopes_fit_the_accepted_boundary(cell_name, deploymen room = MAX_BODY_BYTES - max(len(json.dumps(body)) for body in bodies) - 6 * cell.ORG_MAX_BYTES value["fields"]["oidc_client_id"] = "x" * room org = "\x01" * cell.ORG_MAX_BYTES - assert cell.named_org(org) == org and readable(value) and cell.NAME.fullmatch(cell_name) + assert ( + cell.named_org(org) == org + and cell.readable_fingerprint(value) + and cell.NAME.fullmatch(cell_name) + ) for body in bodies: body["org"] = org assert ( diff --git a/tests/unit/test_cell_observation_census.py b/tests/unit/test_cell_observation_census.py new file mode 100644 index 0000000000..fce91e471b --- /dev/null +++ b/tests/unit/test_cell_observation_census.py @@ -0,0 +1,334 @@ +"""A bounded attempt retains every process obligation across complete reads.""" + +from __future__ import annotations + +import copy +import json +from uuid import UUID + +import pytest + +from server.core import cell +from server.core.cell_ch import Refused +from server.insights.domain import cell_observations as domain +from server.insights.store import cell_observations as store + +KEYS = "a" * 64 +PROOF = "host|boot|container|namespace|123|456|||cell-retirement-v1" + + +def participant(n=1, *, state="running", version=1, proof=PROOF): + return { + "participant_id": str(UUID(int=n)), + "state": state, + "version": version, + "host_proof": proof, + } + + +def context(*, first=False): + fp = cell.as_value({"deployment_id": "dep"}) + return domain.Context("later", first, (fp,), (fp,), KEYS) + + +def observation(n=1, *, role="server", age=0, generation=1): + value = { + **cell.as_value({"deployment_id": "dep"}), + "participant_id": str(UUID(int=n)), + "marked": True, + "serves": role == "server", + "router": False, + "attach_observation": { + "protocol": 1, + "role": role, + "participant_id": str(UUID(int=n)), + "host_proof": PROOF, + "keys_fp": KEYS, + "accepting": True, + "period_ms": 1000, + }, + } + raw = json.dumps(value) + return store.observation( + { + "instance_id": UUID(int=n).hex, + "cell": "later", + "tenancy_mode": "enforced", + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "age_ms": age, + "generation": generation, + } + ) + + +def instance(n=1, **changes): + return {UUID(int=n).hex: {"keys_fp": KEYS, "serving": 1, "stopped": 0, "age_ms": 0, **changes}} + + +def test_server_and_worker_join_the_same_complete_attempt(): + attempt = domain.Attempt() + rows = {p["participant_id"]: p for p in (participant(), participant(2))} + attempt.capture(rows) + values = {v["instance_id"]: v for v in (observation(), observation(2, role="worker"))} + for value in values.values(): + attempt.retain_observation(value) + assert attempt.judge(context(), rows, values, instance()) == ((), KEYS) + assert len(attempt.participants) == len(attempt.observations) == 2 + + +@pytest.mark.parametrize( + "fault", + ["stale", "future", "not-accepting", "missing", "disappeared", "stopped", "no-instance"], +) +def test_unknown_or_nonaccepting_processes_remain_obligations(fault): + attempt = domain.Attempt() + row, value = participant(), observation() + rows = {row["participant_id"]: row} + attempt.capture(rows) + if fault == "stale": + value["age_ms"] = 3001 + elif fault == "future": + value["age_ms"] = -1 + elif fault == "not-accepting": + value["value"]["attach_observation"]["accepting"] = False + values = {} if fault == "missing" else {value["instance_id"]: value} + for observed in values.values(): + attempt.retain_observation(observed) + current = {} if fault == "disappeared" else rows + instances = {} if fault == "no-instance" else instance(stopped=int(fault == "stopped")) + pending, _ = attempt.judge(context(), current, values, instances) + assert row["participant_id"] in pending + assert row["participant_id"] in attempt.participants + + +@pytest.mark.parametrize( + "fault", + [ + "protocol", + "bool-period", + "long-period", + "birth", + "uuid", + "worker-instance", + "cell", + "marker", + "fingerprint", + "keys", + "router", + ], +) +def test_inconsistent_observations_refuse(fault): + attempt = domain.Attempt() + row, value = ( + participant(), + observation(role="worker" if fault == "worker-instance" else "server"), + ) + rows = {row["participant_id"]: row} + attempt.capture(rows) + claim = value["value"]["attach_observation"] + if fault == "protocol": + claim["protocol"] = 2 + elif fault == "bool-period": + claim["period_ms"] = True + elif fault == "long-period": + claim["period_ms"] = 60001 + elif fault == "birth": + claim["host_proof"] = PROOF.replace("|123|", "|124|") + elif fault == "uuid": + claim["participant_id"] = str(UUID(int=2)) + elif fault == "cell": + value["cell"] = "other" + elif fault == "marker": + value["value"]["marked"] = False + elif fault == "fingerprint": + value["value"]["fields"]["deployment_id"] = "other" + elif fault == "keys": + claim["keys_fp"] = "b" * 64 + with pytest.raises(Refused): + attempt.retain_observation(value) + attempt.judge( + context(first=fault == "router"), rows, {value["instance_id"]: value}, instance() + ) + + +def test_retirement_closes_only_the_same_captured_birth_and_keeps_its_budget(): + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + closed = participant(state="retired", version=2) + attempt.capture({closed["participant_id"]: closed}) + before = copy.deepcopy(attempt.participants) + assert attempt.judge(context(), {}, {}, {}) == (("processes",), None) + assert attempt.participants == before and len(before) == 1 + with pytest.raises(Refused, match="birth"): + bad = participant(state="retired", version=3, proof=PROOF.replace("|123|", "|124|")) + attempt.capture({bad["participant_id"]: bad}) + assert attempt.participants == before + + +def test_participant_limit_is_cumulative_including_closed_history(): + attempt = domain.Attempt() + for n in range(1, 1025): + row = participant(n, state="retired") + attempt.capture({row["participant_id"]: row}) + size = attempt.retained_bytes + attempt.capture({}) + for row in list(attempt.participants.values()): + current = row["latest"] + attempt.capture({current["participant_id"]: current}) + assert len(attempt.participants) == 1024 and attempt.retained_bytes == size + row = participant(1025) + with pytest.raises(Refused, match="identity limit"): + attempt.capture({row["participant_id"]: row}) + assert len(attempt.participants) == 1024 and attempt.retained_bytes == size + + +def test_observation_limit_is_cumulative_without_evicting_closed_rows(): + attempt = domain.Attempt() + for n in range(1, 1025): + row = participant(n, state="retired") + attempt.capture({row["participant_id"]: row}) + attempt.retain_observation(observation(n)) + before = copy.deepcopy(attempt.observations) + # A different observation identity for an already closed participant is still new history. + value = observation(1) + value["instance_id"] = UUID(int=1025).hex + with pytest.raises(Refused): + attempt.retain_observation(value) + assert len(attempt.observations) == 1024 and attempt.observations == before + + +def test_retained_growth_is_checked_before_replacement(monkeypatch): + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + before = copy.deepcopy(attempt.participants) + monkeypatch.setattr(domain, "METADATA_BYTES", attempt.retained_bytes) + row = participant(version=100000) + with pytest.raises(Refused, match="byte limit"): + attempt.capture({row["participant_id"]: row}) + assert attempt.participants == before + + +def test_repeated_observations_retain_only_latest_evidence_and_check_generation(): + attempt = domain.Attempt() + row, value = participant(), observation() + attempt.capture({row["participant_id"]: row}) + attempt.retain_observation(value) + size = attempt.retained_bytes + attempt.retain_observation(copy.deepcopy(value)) + assert attempt.retained_bytes == size and len(attempt.observations) == 1 + changed = copy.deepcopy(value) + changed["value"]["marked"] = False + with pytest.raises(Refused, match="equal generations"): + attempt.retain_observation(changed) + changed = copy.deepcopy(value) + changed["generation"] = 0 + with pytest.raises(Refused, match="regressed"): + attempt.retain_observation(changed) + assert attempt.retained_bytes == size + + +def test_a_future_replica_cannot_be_hidden_by_a_fresh_replica(): + value = observation() + raw = json.dumps(value["value"]) + rows = [ + { + "h": host, + "instance_id": value["instance_id"], + "cell": "later", + "cell_bytes": 5, + "tenancy_mode": "enforced", + "tenancy_mode_bytes": 8, + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "generation": 1, + "age_ms": age, + } + for host, age in (("a", 0), ("b", -1)) + ] + attempt = domain.Attempt() + row = participant() + attempt.capture({row["participant_id"]: row}) + merged = store._collapse_observations(rows, attempt) + assert merged[value["instance_id"]]["age_ms"] == -1 + + +@pytest.mark.parametrize("fault", ["replica", "logical", "unknown-host"]) +def test_replica_and_logical_cap_plus_one_refuse_before_latest_selection(fault): + from server.core.cell_ch import Topology + from server.insights.store.provider_participants import bounded_participants + + count = 1 if fault == "unknown-host" else 1025 + rows = [ + { + **participant(n), + "proof_bytes": len(PROOF.encode()), + "h": "outside" + if fault == "unknown-host" + else "a" + if fault == "replica" + else ("a" if n % 2 else "b"), + } + for n in range(1, count + 1) + ] + with pytest.raises(Refused): + bounded_participants( + rows, Topology("main", 1, (("a", 9000), ("b", 9000))), {"a": "a", "b": "b"} + ) + + +def test_replica_duplication_does_not_consume_the_logical_identity_allowance(): + from server.core.cell_ch import Topology + from server.insights.domain.cell_participants import latest + from server.insights.store.provider_participants import bounded_participants + + rows = [ + {**participant(n), "proof_bytes": len(PROOF.encode()), "h": host} + for host in ("a", "b") + for n in range(1, 1025) + ] + selected = bounded_participants( + rows, Topology("main", 1, (("a", 9000), ("b", 9000))), {"a": "a", "b": "b"} + ) + assert len(selected) == 2048 and len(latest(selected)) == 1024 + + +@pytest.mark.parametrize( + ("field", "nondefault"), + [("deployment_id", "dep"), ("public_host", "cell.example.test"), ("configured_orgs", ["acme"])], +) +@pytest.mark.parametrize("pair", ["own", "directory"]) +def test_omitted_observation_field_cannot_match_a_nondefault_pair(field, nondefault, pair): + row, value = participant(), observation(role="worker") + value["value"]["fields"] = {} + default = cell.as_value(cell.DEFAULTS) + accepted = cell.as_value({field: nondefault}) + context = domain.Context( + "later", + False, + (accepted if pair == "own" else default,), + (accepted if pair == "directory" else default,), + KEYS, + ) + attempt = domain.Attempt() + rows = {row["participant_id"]: row} + attempt.capture(rows) + attempt.retain_observation(value) + with pytest.raises(Refused, match="fingerprint disagrees"): + attempt.judge(context, rows, {value["instance_id"]: value}, {}) + + +def test_omitted_default_fields_preserve_unknown_observation_fields(): + row, value = participant(), observation(role="worker") + fields = {"future_field": {"enabled": True}} + value["value"]["fields"] = copy.deepcopy(fields) + accepted = cell.as_value({**cell.DEFAULTS, **fields}) + context = domain.Context("later", False, (accepted,), (accepted,), KEYS) + attempt = domain.Attempt() + rows = {row["participant_id"]: row} + attempt.capture(rows) + attempt.retain_observation(value) + assert attempt.judge(context, rows, {value["instance_id"]: value}, {}) == ((), KEYS) + assert value["value"]["fields"] == fields diff --git a/tests/unit/test_cell_observation_census_stream.py b/tests/unit/test_cell_observation_census_stream.py new file mode 100644 index 0000000000..0c7c967f0d --- /dev/null +++ b/tests/unit/test_cell_observation_census_stream.py @@ -0,0 +1,245 @@ +"""The complete owning reader uses real SQL and the bounded raw transport on fakes.""" + +from __future__ import annotations + +import json +import time + +import pytest + +from server.core.cell_ch import Refused +from server.insights.domain.cell_observations import Attempt +from server.insights.store import cell_observations as store +from server.settings import settings +from tests.unit.test_cell_observation_census import context, instance, observation, participant +from tests.unit.test_cell_scan import FakeCH + +HOSTS = ("os-0", "os-1") + + +def each(rows): + return [{"h": host, **row} for host in HOSTS for row in rows] + + +def fixtures(*, problem=""): + ch = FakeCH() + ch.on( + "FROM system.clusters", + [ + {"host_name": f"ch-{n}", "port": 9000, "shard_num": 1} + for n in range(65 if problem == "replica-cap" else 2) + ], + ) + ch.on("SHOW GRANTS FINAL", [{"g": "GRANT ALL ON *.* TO attach"}]) + ch.on("currentUser()", [{"u": "attach"}]) + ch.on( + "groupArrayIf(host_name, is_local)", + [{"h": host, "names": [f"ch-{n}"]} for n, host in enumerate(HOSTS)], + ) + ch.on("system.processes", each([{"user": "attach"}])) + ch.on("system.users", each([{"name": "attach"}])) + ch.on( + "system.grants", + each( + [ + { + "user_name": "attach", + "role_name": None, + "access_type": "ALL", + "database": None, + "table": None, + } + ] + ), + ) + ch.on("system.role_grants", []) + ch.on("system.row_policies", []) + ch.on("SYSTEM SYNC REPLICA", [{"host": f"ch-{n}", "status": 0} for n in range(2)]) + ch.on( + "system.replicas", + [ + { + "h": host, + "database": "loom", + "table": "_provider_participants", + "zookeeper_path": "/same", + "replica_name": f"r{n}", + "replica_is_active": {"r0": 1, "r1": 1}, + } + for n, host in enumerate(HOSTS) + ], + ) + return _data_routes(ch, problem) + + +def _data_routes(ch, problem): + ch.on("engine FROM", each([{"engine": "ReplicatedReplacingMergeTree"}])) + ch.on( + "engine, engine_full FROM", + each( + [ + { + "name": name, + "engine": "ReplicatedReplacingMergeTree" + if name == "_provider_participants" + else "KeeperMap", + "engine_full": "KeeperMap('/" + name + "')", + } + for name in store._TABLES + ] + ), + ) + columns = each( + [ + {"table": table, "name": name, "type": kind} + for table, cols in store._COLUMNS.items() + for name, kind in cols.items() + ] + ) + if problem == "column": + columns[0]["type"] = "String" + ch.on("system.columns", columns) + count = 0 + + def participants(sql, params): + nonlocal count + count += 1 + rows = [participant(), participant(2)] + if problem == "birth-during-join" and count >= 2: + rows.append(participant(3)) + if problem == "lower-conflict": + rows = [ + participant(version=2), + participant(version=1), + participant(state="drained", version=1), + ] + return each([{**row, "proof_bytes": len(row["host_proof"].encode())} for row in rows]) + + ch.on( + "proof_bytes FROM", + OSError("replica unavailable") if problem == "unavailable" else participants, + ) + beats = [] + for value in (observation(), observation(2, role="worker")): + raw = json.dumps(value["value"]) + beats.append( + { + "instance_id": value["instance_id"], + "cell": "later", + "cell_bytes": 5, + "tenancy_mode": "enforced", + "tenancy_mode_bytes": 8, + "fingerprint": raw, + "fingerprint_bytes": len(raw.encode()), + "age_ms": 0, + "generation": 1, + } + ) + ch.on("`cell_heartbeats`", each(beats)) + ch.on( + "`instances`", + each( + [ + { + "instance_id": key, + **value, + "keys_fp_bytes": len(value["keys_fp"]), + "generation": 1, + } + for key, value in instance().items() + ] + ), + ) + return ch + + +async def test_complete_replica_union_reads_workers_and_rechecks_births(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch, attempt = fixtures(), Attempt() + deadline = time.monotonic() + 10 + result = await store.read_cell(ch, attempt, context(), deadline=deadline) + assert result.pending == () and result.keys_fp == "a" * 64 + assert len(result.participants) == len(result.observations) == 2 + reads = ch.sql("raw_stream") + participant_reads = [sql for sql in reads if "proof_bytes FROM" in sql] + assert len(participant_reads) == 2 and all(" FINAL" not in sql for sql in participant_reads) + beats = next(sql for sql in reads if "`cell_heartbeats`" in sql) + assert "dateDiff('millisecond', census_observation.heartbeat_at, now64(3))" in beats + assert " WHERE " not in beats and "LIMIT 1025 BY h" in beats + assert all( + bounds["skip_unavailable_shards"] == 0 + for kind, _, bounds in ch.calls + if kind == "raw_stream" + ) + retained = attempt.retained_bytes + repeated = await store.read_cell(fixtures(), attempt, context(), deadline=deadline) + assert repeated.pending == () and attempt.retained_bytes == retained + assert repeated.raw_bytes == result.raw_bytes + + +async def test_birth_during_join_becomes_an_obligation_not_an_omission(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch, attempt = fixtures(problem="birth-during-join"), Attempt() + result = await store.read_cell(ch, attempt, context(), deadline=time.monotonic() + 10) + extra = participant(3)["participant_id"] + assert extra in result.pending and extra in attempt.participants + assert len(result.participants) == 3 + + +@pytest.mark.parametrize("problem", ["replica-cap", "column", "lower-conflict", "unavailable"]) +async def test_unverified_complete_census_never_returns_a_snapshot(monkeypatch, problem): + monkeypatch.setattr(settings, "ch_cluster", "main") + ch = fixtures(problem=problem) + with pytest.raises(OSError if problem == "unavailable" else Refused): + await store.read_cell(ch, Attempt(), context(), deadline=time.monotonic() + 10) + # A failed all-replica participant read is never repeated against just this server. + if problem == "unavailable": + reads = [sql for sql in ch.sql("raw_stream") if "proof_bytes FROM" in sql] + assert len(reads) == 1 and "clusterAllReplicas" in reads[0] + + +@pytest.mark.parametrize("field", ["oidc_client_id", "future"]) +async def test_complete_reader_accepts_matching_large_fingerprint_fields(monkeypatch, field): + from dataclasses import replace + + from server.core import cell + + monkeypatch.setattr(settings, "ch_cluster", "main") + accepted = cell.as_value({"deployment_id": "dep", field: "x" * 3000}) + assert cell.readable_fingerprint(accepted) + assert 2048 < len(accepted["fields"][field].encode()) < 64 * 1024 + configured = replace(context(), own_pair=(accepted,), directory_pair=(accepted,)) + configured.validate() + ch = fixtures() + for needle, rows in ch.routes: + if needle == "`cell_heartbeats`": + for row in rows: + value = json.loads(row["fingerprint"]) + value["fields"][field] = accepted["fields"][field] + row["fingerprint"] = json.dumps(value) + row["fingerprint_bytes"] = len(row["fingerprint"].encode()) + result = await store.read_cell(ch, Attempt(), configured, deadline=time.monotonic() + 10) + assert result.pending == () and result.keys_fp == "a" * 64 + assert { + value["value"]["attach_observation"]["role"] for value in result.observations.values() + } == {"server", "worker"} + assert all( + value["value"]["fields"][field] == accepted["fields"][field] + for value in result.observations.values() + ) + + +async def test_complete_reader_still_refuses_oversized_nonfingerprint_metadata(monkeypatch): + monkeypatch.setattr(settings, "ch_cluster", "main") + configured = context() + configured.validate() + ch = fixtures() + for needle, rows in ch.routes: + if needle == "`cell_heartbeats`": + for row in rows: + value = json.loads(row["fingerprint"]) + value["metadata_note"] = "x" * 2049 + row["fingerprint"] = json.dumps(value) + row["fingerprint_bytes"] = len(row["fingerprint"].encode()) + with pytest.raises(Refused, match="process metadata exceeds its field bound"): + await store.read_cell(ch, Attempt(), configured, deadline=time.monotonic() + 10) diff --git a/tests/unit/test_cell_trigger_resource_census.py b/tests/unit/test_cell_trigger_resource_census.py index fb3cf02058..75778534b7 100644 --- a/tests/unit/test_cell_trigger_resource_census.py +++ b/tests/unit/test_cell_trigger_resource_census.py @@ -7,6 +7,7 @@ import pytest from scripts.gates.testsupport.cell_trigger import fixture +from scripts.gates.testsupport.cell_trigger.owned_children import OwnedChildren @pytest.fixture @@ -82,6 +83,145 @@ def read(path, *args, **kwargs): return owner, owned, clock +@pytest.fixture +def rss_census(census, monkeypatch): + owner, owned, clock = census + original_read, original_identity = Path.read_text, owned.identify + gone, failures = set(), {} + + def read(path, *args, **kwargs): + if path.parent.parent == Path("/proc") and path.name == "status": + pid = int(path.parent.name) + if pid in failures: + return failures[pid]() + return "Name:\tfixture\nVmRSS:\t1024 kB\n" + return original_read(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + owned.identify = lambda pid: None if pid in gone else original_identity(pid) + owned.measure = OwnedChildren.rss + return owner, owned, clock, gone, failures + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_vanished_rss_read_requires_a_new_complete_forest(rss_census, error): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise error("gone during RSS read") + + failures[300] = vanish + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 2 + assert owned.sampled_pids.count(200) == 2 + assert 400 in owned.sampled_pids + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +def test_a_live_process_with_unreadable_rss_never_refreshes_the_sample(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS unavailable while identity remains present") + + failures[300] = unreadable + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_a_readable_rss_forest_keeps_the_existing_admission_path(rss_census): + owner, owned, _, _, _ = rss_census + owner.sample() + assert owner.first is None + assert owner.sampled == 10.0 + assert owned.round == 1 + assert set(owned.sampled_pids) == {100, 200, 300} + + +@pytest.mark.parametrize("error", [FileNotFoundError, ProcessLookupError]) +@pytest.mark.parametrize("pid", [100, 200]) +def test_rss_root_loss_does_not_retry_or_refresh(rss_census, error, pid): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(pid) + raise error("root disappeared during RSS read") + + failures[pid] = vanish + owner.sample() + assert owner.first is not None + assert owner.sampled == 7.0 + assert owned.round <= 1 + + +@pytest.mark.parametrize("error", [PermissionError, OSError, ValueError]) +def test_other_rss_read_errors_remain_refusals(rss_census, error): + owner, owned, _, _, failures = rss_census + + def unreadable(): + raise error("RSS read failed without verified disappearance") + + failures[300] = unreadable + owner.sample() + assert isinstance(owner.first, error) + assert owner.sampled == 7.0 + assert owned.round == 1 + + +def test_new_forest_still_checks_the_next_child_rss(rss_census): + owner, owned, _, gone, failures = rss_census + + def vanish(): + gone.add(300) + raise ProcessLookupError("gone during RSS read") + + failures[300] = vanish + failures[400] = lambda: f"VmRSS:\t{fixture.POLICY['stop_process_rss_bytes'] // 1024 + 1} kB\n" + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 2 + assert 400 in owned.sampled_pids + + +def test_rss_disappearance_does_not_reset_the_census_deadline(rss_census): + owner, owned, clock, gone, failures = rss_census + + def vanish(): + gone.add(300) + clock.now = 13.0 + raise ProcessLookupError("gone at the census deadline") + + failures[300] = vanish + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 1 + assert 400 not in owned.sampled_pids + + +def test_repeated_rss_disappearance_keeps_the_three_forest_bound(rss_census): + owner, owned, _, gone, failures = rss_census + owned.trees = [[300], [400], [500], [600]] + + def vanish(pid): + gone.add(pid) + raise ProcessLookupError("another descendant vanished") + + failures.update({pid: lambda pid=pid: vanish(pid) for pid in (300, 400, 500)}) + owner.sample() + assert isinstance(owner.first, fixture.Refused) + assert owner.sampled == 7.0 + assert owned.round == 3 + assert 600 not in owned.sampled_pids + + @pytest.mark.parametrize("stage", ["before", "after", "children"]) def test_a_vanished_descendant_restarts_the_whole_forest(census, stage): owner, owned, _ = census diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py index 353dc1e2cf..76bfb9028a 100644 --- a/tests/unit/test_cell_update_participants.py +++ b/tests/unit/test_cell_update_participants.py @@ -195,8 +195,8 @@ async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): from types import SimpleNamespace from unittest.mock import AsyncMock - from server.cli import cell_update_participants as module from server.core.cell_ch import Topology + from server.insights.store import provider_participants as module events = [] topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) @@ -250,8 +250,8 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey from types import SimpleNamespace from unittest.mock import AsyncMock - from server.cli import cell_update_participants as module from server.core.cell_ch import Topology + from server.insights.store import provider_participants as module topology = Topology( "replicas", @@ -296,6 +296,6 @@ async def per_host(*args): module.cell_sync, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) ) query = AsyncMock(side_effect=AssertionError("must not read partial census")) - with pytest.raises((module.cell_control.Refused, OSError)): + with pytest.raises((module.Refused, OSError)): await module.census(SimpleNamespace(query_all=query)) query.assert_not_awaited() diff --git a/tests/unit/test_ch_bounded_stream.py b/tests/unit/test_ch_bounded_stream.py new file mode 100644 index 0000000000..5e861d1023 --- /dev/null +++ b/tests/unit/test_ch_bounded_stream.py @@ -0,0 +1,101 @@ +"""The canonical byte reader rejects malformed or truncated raw streams.""" + +from __future__ import annotations + +import time +from contextlib import asynccontextmanager + +import pytest + +from server.core.ch_bounded import BoundedReadError, read_rows +from server.insights.domain.cell_observations import Attempt +from server.insights.store.cell_observations import _Reads + + +class Stream: + def __init__(self, chunks): + self.chunks = chunks + self.closed = 0 + + @asynccontextmanager + async def raw_stream(self, sql, params=None, *, settings=None): + async def chunks(): + for chunk in self.chunks: + yield chunk + + try: + yield chunks() + finally: + self.closed += 1 + + +@pytest.mark.parametrize( + "chunks", + [ + [b'{"x":1,"x":2}\n'], + [b'{"x":NaN}\n'], + [b'{"x":1e999}\n'], + [b'{"x":1}'], + [b'{"x":"\xff"}\n'], + [b"[]\n"], + [b"{}\n{}\n"], + ], +) +async def test_malformed_incomplete_or_extra_rows_fail_closed_and_close(chunks): + ch = Stream(chunks) + with pytest.raises(BoundedReadError): + await read_rows( + "SELECT 1", + {}, + ch=ch, + deadline=time.monotonic() + 1, + row_bytes=32, + total_bytes=64, + max_rows=1, + tag="cell.attach.census", + ) + assert ch.closed == 1 + + +async def test_chunk_boundaries_and_exact_byte_allowances_preserve_rows(): + ch = Stream([b'{"', b'x":1}', b"\n"]) + assert await read_rows( + "SELECT 1", + {}, + ch=ch, + deadline=time.monotonic() + 1, + row_bytes=7, + total_bytes=8, + max_rows=1, + tag="cell.attach.census", + ) == [{"x": 1}] + assert ch.closed == 1 + + +async def test_complete_read_aggregates_stream_bytes_but_next_poll_retains_only_obligations( + monkeypatch, +): + from server.insights.domain import cell_observations as domain + + deadline = time.monotonic() + 1 + attempt = Attempt() + monkeypatch.setattr(domain, "METADATA_BYTES", 8) + ch = Stream([b'{"x":1}\n']) + reader = _Reads(ch, attempt, deadline) + assert await reader.query_all("SELECT 1") == [{"x": 1}] + with pytest.raises(BoundedReadError): + await reader.query_all("SELECT 1") + next_poll = _Reads(ch, attempt, deadline) + assert await next_poll.query_all("SELECT 1") == [{"x": 1}] + assert attempt.deadline == deadline and next_poll.raw_bytes_used == 8 + assert attempt.retained_bytes == 0 + + +async def test_next_poll_cannot_reset_the_original_workflow_allowance(): + from server.core.cell_ch import Refused + + attempt = Attempt() + deadline = time.monotonic() + 1 + _Reads(Stream([]), attempt, deadline) + with pytest.raises(Refused, match="reset"): + _Reads(Stream([]), attempt, deadline + 1)