From 99e372aae64fea8523e96d2f1d2bf2fd86bad392 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 18:43:02 +0000 Subject: [PATCH] Cell checks share topology and catalog helpers Move complete topology, catalog and synchronization implementations below CLI owners so server readers can reuse the same checks. Preserve SQL, refusal behavior and assertion contracts. Signed-off-by: zlareb1 --- docs/architecture/cells.md | 2 +- server/cli/cell_attach.py | 9 +- server/cli/cell_attach_beats.py | 2 +- server/cli/cell_attach_pg.py | 2 +- server/cli/cell_control.py | 4 +- server/cli/cell_place.py | 2 +- server/cli/cell_scan.py | 155 +----- server/cli/cell_scim.py | 2 +- server/cli/cell_update_participants.py | 19 +- server/cli/cell_writers.py | 521 +----------------- server/cli/doctor_cell.py | 7 +- server/{cli => core}/cell_ch.py | 2 +- server/core/cell_sync.py | 128 +++++ server/insights/store/cell_catalog.py | 500 +++++++++++++++++ tests/integration/test_cell_attach.py | 8 +- tests/integration/test_cell_scan.py | 20 +- tests/integration/test_cell_unset_gate_pg.py | 2 +- tests/unit/test_cell_attach.py | 6 +- .../unit/test_cell_attach_serving_identity.py | 2 +- tests/unit/test_cell_scan.py | 18 +- tests/unit/test_cell_scim_precondition.py | 2 +- .../test_cell_scim_precondition_failures.py | 2 +- tests/unit/test_cell_update_participants.py | 16 +- tests/unit/test_cell_update_scim_runtime.py | 2 +- tests/unit/test_cell_writers.py | 45 +- tests/unit/test_scim_review118_regressions.py | 2 +- 26 files changed, 763 insertions(+), 717 deletions(-) rename server/{cli => core}/cell_ch.py (99%) create mode 100644 server/core/cell_sync.py create mode 100644 server/insights/store/cell_catalog.py diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index a78e60bce4..ff46557fb8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1716,7 +1716,7 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, objects and replicas outside the operator's allow lists); - `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); + `server/core/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; - `server/auth/tenancy.py` (`ensure_held`): the tenancy check and the placement call, diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index b2e6b25059..241f9143b5 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -46,8 +46,9 @@ from server.cli import cell_attach_pg as apg from server.cli import cell_control, cell_scan, cell_scim from server.cli._ch import writes -from server.cli.cell_ch import Refusal, Refused, read_topology -from server.core import cell, cell_triggers +from server.core import cell, cell_sync, cell_triggers +from server.core.cell_ch import Refusal, Refused, read_topology +from server.insights.store import cell_catalog from server.settings import settings _WAIT_S = 600 # not a setting: a flag's default @@ -324,7 +325,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: topology = await read_topology(self.ch) if refusals := topology.refusals(): return refusals, topology, {} - names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + names = await cell_catalog.host_names(self.ch, topology) if topology.multi else {} allowed = cell_writers.Allowed( users=frozenset(self.args.ch_allow_user), user_directories=frozenset(self.args.ch_allow_user_directory), @@ -406,7 +407,7 @@ async def begin(self) -> None: # -- the scans --------------------------------------------------------------------- async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: - budget = cell_scan.Budget(self.args.scan_budget) + budget = cell_sync.Budget(self.args.scan_budget) current, nxt = self.own_fp return await cell_scan.scan( self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index 57cc73bfd1..539e2982f3 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -26,8 +26,8 @@ from typing import Any, Literal from server.cli.cell_attach_pg import Wait -from server.cli.cell_ch import Refusal, Refused from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused StoppedUnset = Literal["refuse", "wait"] diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 89717d5699..0ca96ce4c0 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -18,9 +18,9 @@ from functools import partial from typing import TYPE_CHECKING, Any -from server.cli.cell_ch import Refusal, Refused from server.core import cell, cell_triggers, pg from server.core.cell import ATTACH_LOCK_KEY +from server.core.cell_ch import Refusal, Refused if TYPE_CHECKING: from server.cli.cell_control import Owner diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 0fbaade144..53d65b7838 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -13,9 +13,9 @@ from uuid import UUID from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal -from server.cli.cell_ch import Refused as Refused from server.core import pg, retirement +from server.core.cell_ch import Refusal +from server.core.cell_ch import Refused as Refused from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index b564ee0cff..321345618b 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -14,9 +14,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_pg as apg from server.cli._ch import run_connected, writes -from server.cli.cell_ch import Refused from server.cli.cell_control import Owner from server.core import cell, pg +from server.core.cell_ch import Refused from server.router.directory import PLACE_MAX_S from server.settings import settings diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index f66435373f..1f292031b7 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -24,14 +24,13 @@ import asyncio import json import re -import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any -from clickhouse_connect.driver.exceptions import DatabaseError - -from server.cli.cell_ch import ( +from server.core import cell_sync, ch_errors, ch_names, pg +from server.core.cell import ORG_FREE, named_org, org_name_refusal +from server.core.cell_ch import ( TAG, WHOLE, WHOLE_SQL, @@ -42,14 +41,12 @@ per_host, quote, ) -from server.cli.cell_writers import own_refusals -from server.core import ch_errors, ch_names, pg -from server.core.cell import ORG_FREE, named_org, org_name_refusal from server.core.cell_triggers import ORG_TABLES from server.core.ch import query_tag from server.core.ch_errors import error_code, error_line from server.erasure.store.census import classified_tables from server.erasure.store.steps_pg import PG_CENSUS, PG_NO_MEMORY_CONTENT +from server.insights.store.cell_catalog import own_refusals # Tables whose orgs are never placed: rows written before any check (a refused or # unauthenticated request's log row keeps the org it named), and tables a view fills from @@ -91,38 +88,11 @@ # How long past the budget a Postgres try may take before this side gives it up, so the # server's own statement timeout fires first when the server answers. _PG_MARGIN_S = 5.0 # not a setting: a margin over --scan-budget -_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED _UNKNOWN_TABLE_CODES = frozenset({60, 81}) # UNKNOWN_TABLE, UNKNOWN_DATABASE _sleep = asyncio.sleep # the seam the retry tests replace -class Budget: - """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" - - def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: - self.seconds = float(seconds) - self._clock = clock - self._start = clock() - - def elapsed(self) -> float: - return self._clock() - self._start - - def left(self) -> float: - return self.seconds - self.elapsed() - - def require(self, subject: str) -> float: - """The seconds left, or Refused naming `subject` once none are.""" - left = self.left() - if left <= 0: - raise Refused([_out_of_budget(subject, self.seconds)]) - return left - - -def _out_of_budget(subject: str, seconds: float) -> Refusal: - return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") - - def pg_quote(name: str) -> str: return '"' + name.replace('"', '""') + '"' @@ -383,7 +353,7 @@ async def list_tables( ) -> TableList: """The scan's table list in both stores, with every refusal it found, attach's own ClickHouse grants checked first; raises `Refused` when a host answers outside the - cluster's names. `names` is `cell_writers.host_names()` on a cluster of several hosts, + cluster's names. `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty.""" with query_tag(TAG): return await _list_tables(ch, pg_conn, topology, names) @@ -501,7 +471,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: cell_sync.Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -529,7 +499,7 @@ async def pg_orgs(table: str, budget: Budget) -> list[str]: # budget ran out only when there is none left, since the timeout is what was left. timed_out = getattr(failure, "sqlstate", None) == "57014" and budget.left() <= 1 if bound.expired() or timed_out: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None # A connect that finds Postgres down comes as PostgresUnavailableError. if not (pg.is_outage(failure) or isinstance(failure, pg.PostgresUnavailableError)): raise Refused([Refusal(subject, f"the scan failed: {failure}")]) from failure @@ -545,63 +515,6 @@ def _ch_transient(e: BaseException) -> bool: ) -def _on_cluster(topology: Topology) -> str: - return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" - - -async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: - """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries - `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must - answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a - profile's `never_throw` hides none). One that does not finish in time refuses as the - budget when the budget is spent, else with the server's error and the hosts it did not - finish on: a replica that cannot sync.""" - seconds = max(1, int(budget.require(subject))) - # `receive_timeout` bounds SYNC REPLICA's own wait. - bounds = {"max_execution_time": seconds, "receive_timeout": seconds} - what = sql.split(" ON CLUSTER ")[0].split(" `")[0] - try: - if not topology.multi: - await ch.execute(sql, settings=bounds) - return - body = await ch.raw_bytes( - sql, - settings={ - **bounds, - "distributed_ddl_output_mode": "throw", - "distributed_ddl_task_timeout": seconds, - "default_format": "JSONEachRow", - }, - ) - rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") - except Exception as e: # only a timeout is judged here; the rest pass to the caller - if error_code(e) not in _TIMEOUT_CODES: - raise - # The statement was given the budget's whole seconds, so up to one is left over. - if budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} - unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" - raise Refused( - [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] - ) from e - failed = [r for r in rows if int(r["status"]) != 0] - if failed: - raise Refused( - [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] - ) - answered = {str(r.get("host")) for r in rows} - if answered != topology.hosts: - missing = ", ".join(sorted(topology.hosts - answered)) or "none" - raise Refused( - [ - Refusal( - subject, f"the hosts that answered are not the listed ones (missing: {missing})" - ) - ] - ) - - def ch_scan_sql(table: Table, topology: Topology) -> str: """The distinct orgs of `table` as hex (bytes: an org that is not UTF-8 is judged, never decoded into another name); on several hosts, first a row naming the configured host @@ -616,36 +529,6 @@ def ch_scan_sql(table: Table, topology: Topology) -> str: return f"{host} UNION ALL {orgs} FORMAT JSONEachRow" -class BodyError(DatabaseError): - """An error ClickHouse wrote into a response body after it began streaming rows: a - server error as the driver raises one, its text keeping the server's `Code: .`, so it - is judged transient or not, and by code, as any other. `rows` are the rows before it.""" - - def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: - super().__init__(text) - self.rows = list(rows or []) - - -def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: - """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that - is not one (an `{"exception": …}` row, or the text tail a server appends once a - statement fails after its first block), with the rest of the body as its text.""" - lines = list(lines) - rows: list[dict[str, Any]] = [] - for i, line in enumerate(lines): - if not line.strip(): - continue - try: - row = json.loads(line) - except ValueError: - row = None - if not isinstance(row, dict) or key not in row: - exception = row.get("exception") if isinstance(row, dict) else None - raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) - rows.append(row) - return rows - - async def _read_scan( ch: Any, sql: str, params: dict[str, Any], extra: dict[str, Any] ) -> list[dict[str, Any]]: @@ -653,23 +536,25 @@ async def _read_scan( async with ch.raw_stream(sql, params, settings=extra) as stream: async for chunk in stream: chunks.append(chunk) - return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") + return cell_sync._body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs( + ch: Any, table: Table, topology: Topology, budget: cell_sync.Budget +) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) try: # Rows buffered on any replica land before the table is read. - flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{_on_cluster(topology)}" - await _system(ch, flush, topology, budget, subject) + flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{cell_sync._on_cluster(topology)}" + await cell_sync._system(ch, flush, topology, budget, subject) if table.engine.startswith("Replicated"): # Every replica pulls every part acknowledged before now, so the read # through any one of them sees them all. - await _system( + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{_on_cluster(topology)} {table.sql} LIGHTWEIGHT", + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} {table.sql} LIGHTWEIGHT", topology, budget, subject, @@ -695,9 +580,9 @@ async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> if r["k"] == "o" ] # A timeout is the budget only when none is left: a profile's estimate throws early. - if error_code(failure) in _TIMEOUT_CODES and budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - if error_code(failure) in _TIMEOUT_CODES or not _ch_transient(failure): + if error_code(failure) in cell_sync._TIMEOUT_CODES and budget.left() <= 1: + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None + if error_code(failure) in cell_sync._TIMEOUT_CODES or not _ch_transient(failure): raise Refused( [Refusal(subject, f"the scan failed: {error_line(failure)}")] ) from failure @@ -722,7 +607,7 @@ async def scan( ch: Any, tables: TableList, topology: Topology, - budget: Budget, + budget: cell_sync.Budget, configured: Iterable[str] = (), ) -> ScanResult: """The distinct orgs of every table of `tables` and the `configured` orgs; Refused, with diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py index aa0573c347..230b378939 100644 --- a/server/cli/cell_scim.py +++ b/server/cli/cell_scim.py @@ -14,8 +14,8 @@ from server.auth import claims, revocation, scim, scim_people, subject_revocation from server.auth.scim_routes import scim_settings_problem from server.cli import cell_control -from server.cli.cell_ch import WHOLE_SQL, Refusal, Refused from server.core import cell, ch +from server.core.cell_ch import WHOLE_SQL, Refusal, Refused from server.core.datetimes import as_aware_utc from server.settings import settings diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index 26a3fa8c73..b948d47e8e 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -9,9 +9,9 @@ from typing import Any, cast from uuid import UUID -from server.cli import cell_control, cell_scan, cell_writers -from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology -from server.core import cell, ch_names, instances +from server.cli import cell_control +from server.core import cell, cell_sync, ch_names, instances +from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -19,6 +19,7 @@ participant_tag, proves_stopped, ) +from server.insights.store import cell_catalog from server.insights.store import provider_participants as participants @@ -88,8 +89,8 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: raise cell_control._refused("participant census cannot resolve the configured cluster") if problems := topology.refusals(): raise Refused(problems) - names = await cell_writers.host_names(ch, topology) if topology.multi else {} - if problems := (await cell_writers.own_refusals(ch, topology, names))[0]: + names = await cell_catalog.host_names(ch, topology) if topology.multi else {} + if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: raise Refused(problems) database = ch_names.database() tables = await per_host( @@ -122,14 +123,14 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: ), {"db": database}, ) - if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas( + if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( row for value in rows.values() for row in value ): raise cell_control._refused("participant replica set includes an unverified replica") - budget = cell_scan.Budget(cell_control.TRANSACTION_S) - await cell_scan._system( + budget = cell_sync.Budget(cell_control.TRANSACTION_S) + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} " + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", topology, budget, diff --git a/server/cli/cell_writers.py b/server/cli/cell_writers.py index 0aa0c8e0b8..2d7e6345af 100644 --- a/server/cli/cell_writers.py +++ b/server/cli/cell_writers.py @@ -31,37 +31,15 @@ from dataclasses import dataclass from typing import Any -from server.cli.cell_ch import ( - CLUSTER_SQL, - TAG, - WHOLE_SQL, +from server.core.cell_ch import ( WRITERS_TAG, Refusal, - Refused, Topology, databases, per_host, ) -from server.core import ch_names from server.core.ch import query_tag -from server.core.ch_errors import error_code, error_line -from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME - -_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED - -# Each privilege attach needs and the grants that imply it, closest first. -_IMPLIED_BY: dict[str, tuple[str, ...]] = { - "SELECT": ("ALL",), - "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW TABLES": ("SHOW", "ALL"), - "SHOW COLUMNS": ("SHOW", "ALL"), - "REMOTE": ("SOURCES", "ALL"), - "CLUSTER": ("ALL",), - "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), - "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), -} +from server.insights.store import cell_catalog # Grants that write the configured databases or hand that right on: a user holding one on # them, or globally, must be one the operator names. @@ -163,235 +141,34 @@ class Allowed: replicas: frozenset[str] = frozenset() -@dataclass(frozen=True) -class Grant: - """One privilege a user holds: `database` None for a global grant, `table` None for the - whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial - revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" - - access: str - database: str | None = None - table: str | None = None - grant_option: bool = False - revoked: bool = False - - def covers(self, database: str | None) -> bool: - """Whether this grant reaches all of `database` (None: every database).""" - if self.table is not None: - return False - return self.database is None or (database is not None and _db_match(self, database)) - - def touches(self, database: str) -> bool: - """Whether this grant reaches any table of `database`.""" - return self.database is None or _db_match(self, database) - - -def _db_match(grant: Grant, database: str) -> bool: - assert grant.database is not None - if grant.database.endswith("*"): - return database.startswith(grant.database[:-1]) - return grant.database == database - - -def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: - """Whether `grants` give `access` on all of `database` (None: globally).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) - - -def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: - """The partial revokes in `grants` of `access`, or of a grant implying it, on any part - of `database` (None: anywhere).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return [ - g - for g in grants - if g.revoked - and g.access in wanted - and (database is None or g.database is None or _db_match(g, database)) - ] - - -# -- reading grants ---------------------------------------------------------------------- - -_GRANT = re.compile( - r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " - r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" -) -_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") - - -def _split_privileges(text: str) -> list[str]: - """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" - out, depth, start = [], 0, 0 - for i, ch in enumerate(text): - depth += {"(": 1, ")": -1}.get(ch, 0) - if ch == "," and depth == 0: - out.append(text[start:i]) - start = i + 1 - out.append(text[start:]) - return [p.split("(")[0].strip() for p in out if p.strip()] - - -def _target(on: str) -> tuple[str | None, str | None] | str: - """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a - `READ` grant names (`REMOTE`) as itself.""" - if on == "*": - return (None, None) - parts = [a or b for a, b in _IDENT.findall(on)] - if len(parts) != 2: - return on - db, table = parts - return (None if db == "*" else db, None if table == "*" else table) - - -def parse_show_grants(lines: Iterable[str]) -> list[Grant]: - """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its - roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the - revokes of a grant option alone are left out.""" - out = [] - for line in lines: - m = _GRANT.match(line.strip()) - if m is None or m["privs"].startswith("GRANT OPTION FOR "): - continue - target = _target(m["on"]) - option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" - for access in _split_privileges(m["privs"]): - if isinstance(target, str): - # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it - # as written. - if access in ("READ", "WRITE"): - out.append(Grant(target, grant_option=option, revoked=revoked)) - continue - out.append(Grant(access, target[0], target[1], option, revoked)) - return out - - -def resolve_grants( - user: str, - grant_rows: Iterable[Mapping[str, Any]], - role_rows: Iterable[Mapping[str, Any]], - *, - default_only: bool = False, -) -> list[Grant]: - """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every - role it holds followed (default or not, since it can set any). `default_only` follows - only the user's default roles, the ones a new session has (attach's own reads run in - one), and every role those hold: `granted_role_is_default` marks the user's own grants, - while a role's grant of another always passes it on.""" - # A user and a role may share a name: each holder is keyed by its kind. - granted: dict[tuple[str, str], set[str]] = {} - for r in role_rows: - holder = ( - ("user", str(r["user_name"])) - if r.get("user_name") - else ("role", str(r.get("role_name"))) - ) - if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: - continue - granted.setdefault(holder, set()).add(str(r["granted_role_name"])) - roles: set[str] = set() - todo = list(granted.get(("user", user), ())) - while todo: - role = todo.pop() - if role not in roles: - roles.add(role) - todo.extend(granted.get(("role", role), ())) - out = [] - for r in grant_rows: - if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): - out.append(_row_grant(r)) - return out - - -def _row_grant(r: Mapping[str, Any]) -> Grant: - """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object - `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database - as a prefix.""" - access = str(r["access_type"]) - if access in ("READ", "WRITE") and r.get("access_object"): - access = str(r["access_object"]) - database = None if r.get("database") is None else str(r["database"]) - if database is not None and r.get("is_wildcard") and not database.endswith("*"): - database += "*" - return Grant( - access, - database, - None if r.get("table") is None else str(r["table"]), - bool(r.get("grant_option")), - bool(r.get("is_partial_revoke")), - ) - - # -- judging ----------------------------------------------------------------------------- -def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: - """What attach's own user lacks to read what the check and the scan read.""" - needs: list[tuple[str, str | None]] = [ - ("SELECT", "system"), - ("SHOW USERS", None), - ("SHOW ROLES", None), - ("SHOW ROW POLICIES", None), - ("SHOW TABLES", None), - ("SHOW COLUMNS", None), - ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), - ] - for db in dbs: - needs.append(("SELECT", db)) - if topology.hosts: - needs.append(("SYSTEM SYNC REPLICA", db)) - if topology.multi: - needs += [("REMOTE", None), ("CLUSTER", None)] - out = [] - for access, scope in needs: - if not holds(grants, access, scope): - out.append(_label(access, scope)) - elif revokes := narrowing(grants, access, scope): - # A revoke hides part of what the checks read, which reads short silently. - out.append( - f"{_label(access, scope)} (narrowed by REVOKE " - + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) - + ")" - ) - return out - - -def _label(access: str, db: str | None) -> str: - if access == "REMOTE": - return "READ ON REMOTE" - return f"{access} ON {db}.*" if db else f"{access} ON *.*" - - -def writer_reason(grants: Iterable[Grant], dbs: list[str]) -> str: +def writer_reason(grants: Iterable[cell_catalog.Grant], dbs: list[str]) -> str: """Why a user with `grants` can write the configured databases ('' when it cannot): a grant that writes them from anywhere (`_ANYWHERE_ACCESS`), a writing grant on them or globally, else any grant there it can hand on.""" grants = [g for g in grants if not g.revoked] # a revoke only narrows: ignored for g in grants: if g.access in _ANYWHERE_ACCESS: - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" reaching = [g for g in grants if any(g.touches(db) for db in dbs)] for g in reaching: if g.access in _WRITER_ACCESS or g.access.startswith("ALTER "): - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" for g in reaching: if g.grant_option: - return f"holds {g.access} WITH GRANT OPTION on {_scope(g)}" + return f"holds {g.access} WITH GRANT OPTION on {cell_catalog._scope(g)}" # A row policy created later hides rows from the next scan or check, as a write made # later adds them: on the configured databases or the system tables the checks read. for g in grants: if g.access in _POLICY_ACCESS and (g.touches("system") or any(g.touches(db) for db in dbs)): - return f"holds {g.access} on {_scope(g)}, which can hide rows from the scan" + return ( + f"holds {g.access} on {cell_catalog._scope(g)}, which can hide rows from the scan" + ) return "" -def _scope(g: Grant) -> str: - if g.database is None: - return "*.*" - return f"{g.database}.{g.table or '*'}" - - _TO = re.compile(r"\bTO\s+(?:`((?:[^`\\]|\\.)*)`|(\w+))\s*\.", re.I) _NAME = r"(?:`(?:[^`\\]|\\.)*`|[A-Za-z_]\w*)" # `CREATE … VIEW `: a view's `TO` target follows it, and its name may hold anything. @@ -646,280 +423,19 @@ def _inside_problem( return f"a table of engine {engine}, which reads outside the configured databases" -def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: - """Per replicated table (`db.table`), the replicas Keeper lists that no host of the - cluster is (`replica_is_active` keys against every host's `replica_name`, by - `zookeeper_path`).""" - ours: dict[str, set[str]] = {} - listed: dict[str, set[str]] = {} - names: dict[str, str] = {} - for r in rows: - path = str(r["zookeeper_path"]) - ours.setdefault(path, set()).add(str(r["replica_name"])) - listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) - names.setdefault(path, f"{r['database']}.{r['table']}") - out = {} - for path, keys in listed.items(): - if extra := keys - ours[path]: - out[names[path]] = extra - return out - - # -- the check --------------------------------------------------------------------------- -async def _own_user(ch: Any) -> str: - row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") - return str((row or {}).get("u") or "") - - -async def _show_grants(ch: Any) -> list[Grant]: - """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no - privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" - # Pinned: a profile's `break` could end the list before its REVOKE lines. - rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") - return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) - - -# Per host, each user it defines and that user's grants. -HostGrants = dict[str, dict[str, list[Grant]]] - - -async def host_names(ch: Any, topology: Topology) -> dict[str, str]: - """Each answering host's OS hostname (an opaque group key) and the one configured name - it answers for in the cluster; Refused unless there is one group per listed host, each - with exactly one name, the names together the listing. The topology's own refusals (two - servers under one host name, several shards) come first, then what attach's own user - lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read - through `clusterAllReplicas`.""" - if refusals := topology.refusals(): - raise Refused(refusals) - with query_tag(TAG): - if lacking := await first_refusals(ch, topology): - raise Refused(lacking) - rows = await ch.query_all( - "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " - "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " - f"GROUP BY h {CLUSTER_SQL}", - {"c": topology.cluster}, - ) - where = f"cluster {topology.cluster}" - refusals = [] - out: dict[str, str] = {} - for r in rows: - names = sorted(str(n) for n in r["names"]) - if not names: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host's address in {where} does not resolve to a local interface", - ) - ) - elif len(names) > 1: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host answers for several entries in {where} ({', '.join(names)}): " - "two local entries, or two hosts with one OS hostname", - ) - ) - else: - out[str(r["h"])] = names[0] - named = sorted(out.values()) - if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): - missing = sorted(topology.hosts - set(named)) - refusals.append( - Refusal( - where, - "the answering hosts are not the listed ones" - + (f" (no answer for {', '.join(missing)})" if missing else ""), - ) - ) - if refusals: - raise Refused(refusals) - return out - - -async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: - """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs - no privilege. It prints the grants of every role the user holds, a role a session does - not have included: this first look names what is missing; the rows decide.""" - if missing := own_problems(await _show_grants(ch), topology, databases()): - own = await _own_user(ch) - return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] - return [] - - -async def own_refusals( - ch: Any, topology: Topology, names: Mapping[str, str] -) -> tuple[list[Refusal], HostGrants]: - """What attach's own user lacks to read what the checks read (without it `system.tables` - and `system.columns` would silently list less than there is), and every user's grants - per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several - hosts, that every host runs the reads as this user; then that user's grants as each host - keeps them; then the row policies, which would hide rows from its scan.""" - dbs = databases() - if lacking := await first_refusals(ch, topology): - return lacking, {} - own = await _own_user(ch) - try: - if topology.multi and (wrong := await _process_users(ch, topology, names, own)): - return wrong, {} - per_user, own_grants = await _host_grants(ch, topology, names, own) - out = [] - for host, grants in sorted(own_grants.items()): - # Each host keeps its own users: attach's must hold the same on every one, - # through its default roles alone, as its sessions have them. - if lacks := own_problems(grants, topology, dbs): - out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) - if not out: - out = await _row_policies(ch, topology, names, own, dbs) - except Exception as e: # only a refused read is judged here; the rest pass to the caller - if error_code(e) != _ACCESS_DENIED: - raise - reason = ( - f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " - "held only through a role that is not a default role does not reach its sessions" - ) - return [Refusal(f"user {own}", reason)], {} - return out, per_user - - -async def _row_policies( - ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] -) -> list[Refusal]: - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " - "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " - "WHERE database IN {dbs:Array(String)} OR database = 'system'", - {"dbs": dbs}, - ) - out = [] - for host, found in sorted(rows.items()): - for subject, why in row_policy_problems(found, own): - out.append(Refusal(subject, f"{why}{_on(host)}")) - return out - - -def _as_loom_writes(r: Mapping[str, Any]) -> bool: - """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in - either list could cover or exclude attach's user without naming it.""" - if r["short_name"] == POLICY_NAME: - return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] - return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] - - -def _applies(r: Mapping[str, Any], user: str) -> bool: - if r.get("apply_to_all"): - return user not in (r.get("apply_to_except") or []) - return user in (r.get("apply_to_list") or []) - - -def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: - """What hides rows from attach's own user, per row policy on a table of the configured - databases or a system table. Any policy on a system table refuses: the checks read them. - Once a table has a row policy, a user that no permissive policy covers reads none of its - rows, and a restrictive one filters them: either shortens the scan silently. Loom's own - pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` - letting every other user read every row) while it covers attach's user as written; every - other policy refuses.""" - tables: dict[str, list[Mapping[str, Any]]] = {} - for r in rows: - tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) - out = [] - for table, policies in sorted(tables.items()): - subject = f"row policy on {table}" - if table.startswith("system."): - # The checks read the system tables: a policy there hides what they read. Loom - # writes none. - out += [ - (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") - for p in policies - ] - continue - others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] - for p in policies: - name = str(p["short_name"]) - if name not in (POLICY_NAME, OTHERS_POLICY_NAME): - out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) - elif not _as_loom_writes(p): - out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) - elif name == POLICY_NAME and _applies(p, own): - out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) - if not any( - not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) - for p in others - ): - out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) - return out - - -async def _process_users( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> list[Refusal]: - """On several hosts: each host runs this read as attach's own user, exactly once.""" - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, user FROM {source:processes} " - "WHERE initial_query_id = initialQueryID()", - ) - out = [] - for host, found in sorted(rows.items()): - users = [str(r["user"]) for r in found] - if users != [own]: - out.append( - Refusal( - f"host {host}", - f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " - "interserver secret, or the cluster's user to attach's", - ) - ) - return out - - -async def _host_grants( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> tuple[HostGrants, dict[str, list[Grant]]]: - """Per host, each user it defines and that user's grants, from `system.grants` and - `system.role_grants` on every topology (every granted role followed, default or not); - and per host the grants of `own` through its default roles alone.""" - users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") - # Every column: `access_object` and `is_wildcard` are read where the server has them. - grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") - roles = await per_host( - ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" - ) - every = { - host: { - str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found - } - for host, found in users.items() - } - own_grants = { - host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users - } - return every, own_grants - - -def _on(host: str) -> str: - return f" on host {host}" if host else "" - - async def check_writers( ch: Any, topology: Topology, names: Mapping[str, str], allowed: Allowed ) -> list[Refusal]: """Every ClickHouse writer of the configured databases the operator has not named. - `names` is `host_names()` on a cluster of several hosts, else empty. Raises + `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty. Raises `cell_ch.Refused` when a host answers outside the cluster's names; every other refusal is returned.""" dbs = databases() with query_tag(WRITERS_TAG): - lacking, per_user = await own_refusals(ch, topology, names) + lacking, per_user = await cell_catalog.own_refusals(ch, topology, names) if lacking: return lacking refusals: list[Refusal] = [] @@ -927,7 +443,10 @@ async def check_writers( for user, grants in sorted(users.items()): if user not in allowed.users and (why := writer_reason(grants, dbs)): refusals.append( - Refusal(f"user {user}", f"{why}{_on(host)}: name it with --ch-allow-user") + Refusal( + f"user {user}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-user", + ) ) refusals += await _directories(ch, topology, names, allowed) refusals += await _objects(ch, topology, names, allowed, dbs) @@ -949,7 +468,8 @@ async def _directories( out.append( Refusal( f"user directory {name}", - f"its type, {kind}, defines users this check cannot read{_on(host)}: " + f"its type, {kind}, defines users this check cannot read" + f"{cell_catalog._on(host)}: " "name it with --ch-allow-user-directory", ) ) @@ -980,7 +500,10 @@ async def _objects( for key, why in sorted(object_problems(found, dbs, functions, topology.cluster).items()): if key not in allowed.objects: out.append( - Refusal(f"object {key}", f"{why}{_on(host)}: name it with --ch-allow-object") + Refusal( + f"object {key}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-object", + ) ) return out @@ -998,7 +521,7 @@ async def _replicas( ) every = [r for found in rows.values() for r in found] out = [] - for table, extra in sorted(extra_replicas(every).items()): + for table, extra in sorted(cell_catalog.extra_replicas(every).items()): for replica in sorted(extra): if f"{table}:{replica}" not in allowed.replicas: out.append( diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 292f03f374..194964832c 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -20,6 +20,7 @@ from server.cli._ch import reads from server.cli.doctor import FAIL, PASS, WARN +from server.core import cell_sync # A doctor run never hangs on a cell's stores: each statement, and each session's connect # and reads together. @@ -263,13 +264,13 @@ async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: from server.cli import cell_scan - from server.cli.cell_ch import Refused, databases, read_topology from server.core import pg + from server.core.cell_ch import Refused, databases, read_topology from server.core.cell_triggers import ORG_TABLES topology = await read_topology(ch) if topology.multi: - from server.cli.cell_writers import host_names + from server.insights.store.cell_catalog import host_names names = await host_names(ch, topology) else: @@ -289,7 +290,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] work.sort(key=lambda w: (w[0], str(w[2]))) - budget = cell_scan.Budget(budget_s) + budget = cell_sync.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] refused: list[str] = [] diff --git a/server/cli/cell_ch.py b/server/core/cell_ch.py similarity index 99% rename from server/cli/cell_ch.py rename to server/core/cell_ch.py index 3da3766379..6262a23176 100644 --- a/server/cli/cell_ch.py +++ b/server/core/cell_ch.py @@ -157,7 +157,7 @@ async def per_host( """The rows of `sql`, which selects `hostName() AS h` from `{source:}`, grouped by the configured name of the host each came from: system table `
` itself on one host (grouped under ''), through `clusterAllReplicas` on several (grouped by `names`, - from `cell_writers.host_names()`).""" + from `cell_catalog.host_names()`).""" m = re.search(r"\{source:(\w+)\}", sql) assert m, sql table = f"system.{m.group(1)}" diff --git a/server/core/cell_sync.py b/server/core/cell_sync.py new file mode 100644 index 0000000000..e723a41784 --- /dev/null +++ b/server/core/cell_sync.py @@ -0,0 +1,128 @@ +"""Complete replica synchronization with the caller's existing scan budget.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable, Iterable +from typing import Any + +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core.cell_ch import Refusal, Refused, Topology, quote +from server.core.ch_errors import error_code, error_line + +_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED + + +class Budget: + """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self._clock = clock + self._start = clock() + + def elapsed(self) -> float: + return self._clock() - self._start + + def left(self) -> float: + return self.seconds - self.elapsed() + + def require(self, subject: str) -> float: + """The seconds left, or Refused naming `subject` once none are.""" + left = self.left() + if left <= 0: + raise Refused([_out_of_budget(subject, self.seconds)]) + return left + + +def _out_of_budget(subject: str, seconds: float) -> Refusal: + return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") + + +def _on_cluster(topology: Topology) -> str: + return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" + + +async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: + """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries + `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must + answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a + profile's `never_throw` hides none). One that does not finish in time refuses as the + budget when the budget is spent, else with the server's error and the hosts it did not + finish on: a replica that cannot sync.""" + seconds = max(1, int(budget.require(subject))) + # `receive_timeout` bounds SYNC REPLICA's own wait. + bounds = {"max_execution_time": seconds, "receive_timeout": seconds} + what = sql.split(" ON CLUSTER ")[0].split(" `")[0] + try: + if not topology.multi: + await ch.execute(sql, settings=bounds) + return + body = await ch.raw_bytes( + sql, + settings={ + **bounds, + "distributed_ddl_output_mode": "throw", + "distributed_ddl_task_timeout": seconds, + "default_format": "JSONEachRow", + }, + ) + rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") + except Exception as e: # only a timeout is judged here; the rest pass to the caller + if error_code(e) not in _TIMEOUT_CODES: + raise + # The statement was given the budget's whole seconds, so up to one is left over. + if budget.left() <= 1: + raise Refused([_out_of_budget(subject, budget.seconds)]) from None + done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} + unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" + raise Refused( + [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] + ) from e + failed = [r for r in rows if int(r["status"]) != 0] + if failed: + raise Refused( + [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] + ) + answered = {str(r.get("host")) for r in rows} + if answered != topology.hosts: + missing = ", ".join(sorted(topology.hosts - answered)) or "none" + raise Refused( + [ + Refusal( + subject, f"the hosts that answered are not the listed ones (missing: {missing})" + ) + ] + ) + + +class BodyError(DatabaseError): + """An error ClickHouse wrote into a response body after it began streaming rows: a + server error as the driver raises one, its text keeping the server's `Code: .`, so it + is judged transient or not, and by code, as any other. `rows` are the rows before it.""" + + def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: + super().__init__(text) + self.rows = list(rows or []) + + +def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: + """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that + is not one (an `{"exception": …}` row, or the text tail a server appends once a + statement fails after its first block), with the rest of the body as its text.""" + lines = list(lines) + rows: list[dict[str, Any]] = [] + for i, line in enumerate(lines): + if not line.strip(): + continue + try: + row = json.loads(line) + except ValueError: + row = None + if not isinstance(row, dict) or key not in row: + exception = row.get("exception") if isinstance(row, dict) else None + raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) + rows.append(row) + return rows diff --git a/server/insights/store/cell_catalog.py b/server/insights/store/cell_catalog.py new file mode 100644 index 0000000000..e973826ae4 --- /dev/null +++ b/server/insights/store/cell_catalog.py @@ -0,0 +1,500 @@ +"""Complete catalog visibility, replica identity and grant checks for cell reads.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from server.core import ch_names +from server.core.cell_ch import ( + CLUSTER_SQL, + TAG, + WHOLE_SQL, + Refusal, + Refused, + Topology, + databases, + per_host, +) +from server.core.ch import query_tag +from server.core.ch_errors import error_code, error_line +from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME + +_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED + +_IMPLIED_BY: dict[str, tuple[str, ...]] = { + "SELECT": ("ALL",), + "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW TABLES": ("SHOW", "ALL"), + "SHOW COLUMNS": ("SHOW", "ALL"), + "REMOTE": ("SOURCES", "ALL"), + "CLUSTER": ("ALL",), + "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), + "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), +} + + +@dataclass(frozen=True) +class Grant: + """One privilege a user holds: `database` None for a global grant, `table` None for the + whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial + revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" + + access: str + database: str | None = None + table: str | None = None + grant_option: bool = False + revoked: bool = False + + def covers(self, database: str | None) -> bool: + """Whether this grant reaches all of `database` (None: every database).""" + if self.table is not None: + return False + return self.database is None or (database is not None and _db_match(self, database)) + + def touches(self, database: str) -> bool: + """Whether this grant reaches any table of `database`.""" + return self.database is None or _db_match(self, database) + + +def _db_match(grant: Grant, database: str) -> bool: + assert grant.database is not None + if grant.database.endswith("*"): + return database.startswith(grant.database[:-1]) + return grant.database == database + + +def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: + """Whether `grants` give `access` on all of `database` (None: globally).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) + + +def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: + """The partial revokes in `grants` of `access`, or of a grant implying it, on any part + of `database` (None: anywhere).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return [ + g + for g in grants + if g.revoked + and g.access in wanted + and (database is None or g.database is None or _db_match(g, database)) + ] + + +_GRANT = re.compile( + r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " + r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" +) + +_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") + + +def _split_privileges(text: str) -> list[str]: + """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" + out, depth, start = [], 0, 0 + for i, ch in enumerate(text): + depth += {"(": 1, ")": -1}.get(ch, 0) + if ch == "," and depth == 0: + out.append(text[start:i]) + start = i + 1 + out.append(text[start:]) + return [p.split("(")[0].strip() for p in out if p.strip()] + + +def _target(on: str) -> tuple[str | None, str | None] | str: + """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a + `READ` grant names (`REMOTE`) as itself.""" + if on == "*": + return (None, None) + parts = [a or b for a, b in _IDENT.findall(on)] + if len(parts) != 2: + return on + db, table = parts + return (None if db == "*" else db, None if table == "*" else table) + + +def parse_show_grants(lines: Iterable[str]) -> list[Grant]: + """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its + roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the + revokes of a grant option alone are left out.""" + out = [] + for line in lines: + m = _GRANT.match(line.strip()) + if m is None or m["privs"].startswith("GRANT OPTION FOR "): + continue + target = _target(m["on"]) + option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" + for access in _split_privileges(m["privs"]): + if isinstance(target, str): + # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it + # as written. + if access in ("READ", "WRITE"): + out.append(Grant(target, grant_option=option, revoked=revoked)) + continue + out.append(Grant(access, target[0], target[1], option, revoked)) + return out + + +def resolve_grants( + user: str, + grant_rows: Iterable[Mapping[str, Any]], + role_rows: Iterable[Mapping[str, Any]], + *, + default_only: bool = False, +) -> list[Grant]: + """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every + role it holds followed (default or not, since it can set any). `default_only` follows + only the user's default roles, the ones a new session has (attach's own reads run in + one), and every role those hold: `granted_role_is_default` marks the user's own grants, + while a role's grant of another always passes it on.""" + # A user and a role may share a name: each holder is keyed by its kind. + granted: dict[tuple[str, str], set[str]] = {} + for r in role_rows: + holder = ( + ("user", str(r["user_name"])) + if r.get("user_name") + else ("role", str(r.get("role_name"))) + ) + if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: + continue + granted.setdefault(holder, set()).add(str(r["granted_role_name"])) + roles: set[str] = set() + todo = list(granted.get(("user", user), ())) + while todo: + role = todo.pop() + if role not in roles: + roles.add(role) + todo.extend(granted.get(("role", role), ())) + out = [] + for r in grant_rows: + if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): + out.append(_row_grant(r)) + return out + + +def _row_grant(r: Mapping[str, Any]) -> Grant: + """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object + `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database + as a prefix.""" + access = str(r["access_type"]) + if access in ("READ", "WRITE") and r.get("access_object"): + access = str(r["access_object"]) + database = None if r.get("database") is None else str(r["database"]) + if database is not None and r.get("is_wildcard") and not database.endswith("*"): + database += "*" + return Grant( + access, + database, + None if r.get("table") is None else str(r["table"]), + bool(r.get("grant_option")), + bool(r.get("is_partial_revoke")), + ) + + +def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: + """What attach's own user lacks to read what the check and the scan read.""" + needs: list[tuple[str, str | None]] = [ + ("SELECT", "system"), + ("SHOW USERS", None), + ("SHOW ROLES", None), + ("SHOW ROW POLICIES", None), + ("SHOW TABLES", None), + ("SHOW COLUMNS", None), + ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), + ] + for db in dbs: + needs.append(("SELECT", db)) + if topology.hosts: + needs.append(("SYSTEM SYNC REPLICA", db)) + if topology.multi: + needs += [("REMOTE", None), ("CLUSTER", None)] + out = [] + for access, scope in needs: + if not holds(grants, access, scope): + out.append(_label(access, scope)) + elif revokes := narrowing(grants, access, scope): + # A revoke hides part of what the checks read, which reads short silently. + out.append( + f"{_label(access, scope)} (narrowed by REVOKE " + + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) + + ")" + ) + return out + + +def _label(access: str, db: str | None) -> str: + if access == "REMOTE": + return "READ ON REMOTE" + return f"{access} ON {db}.*" if db else f"{access} ON *.*" + + +def _scope(g: Grant) -> str: + if g.database is None: + return "*.*" + return f"{g.database}.{g.table or '*'}" + + +def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: + """Per replicated table (`db.table`), the replicas Keeper lists that no host of the + cluster is (`replica_is_active` keys against every host's `replica_name`, by + `zookeeper_path`).""" + ours: dict[str, set[str]] = {} + listed: dict[str, set[str]] = {} + names: dict[str, str] = {} + for r in rows: + path = str(r["zookeeper_path"]) + ours.setdefault(path, set()).add(str(r["replica_name"])) + listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) + names.setdefault(path, f"{r['database']}.{r['table']}") + out = {} + for path, keys in listed.items(): + if extra := keys - ours[path]: + out[names[path]] = extra + return out + + +async def _own_user(ch: Any) -> str: + row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") + return str((row or {}).get("u") or "") + + +async def _show_grants(ch: Any) -> list[Grant]: + """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no + privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" + # Pinned: a profile's `break` could end the list before its REVOKE lines. + rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") + return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) + + +HostGrants = dict[str, dict[str, list[Grant]]] + + +async def host_names(ch: Any, topology: Topology) -> dict[str, str]: + """Each answering host's OS hostname (an opaque group key) and the one configured name + it answers for in the cluster; Refused unless there is one group per listed host, each + with exactly one name, the names together the listing. The topology's own refusals (two + servers under one host name, several shards) come first, then what attach's own user + lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read + through `clusterAllReplicas`.""" + if refusals := topology.refusals(): + raise Refused(refusals) + with query_tag(TAG): + if lacking := await first_refusals(ch, topology): + raise Refused(lacking) + rows = await ch.query_all( + "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " + "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " + f"GROUP BY h {CLUSTER_SQL}", + {"c": topology.cluster}, + ) + where = f"cluster {topology.cluster}" + refusals = [] + out: dict[str, str] = {} + for r in rows: + names = sorted(str(n) for n in r["names"]) + if not names: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host's address in {where} does not resolve to a local interface", + ) + ) + elif len(names) > 1: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host answers for several entries in {where} ({', '.join(names)}): " + "two local entries, or two hosts with one OS hostname", + ) + ) + else: + out[str(r["h"])] = names[0] + named = sorted(out.values()) + if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): + missing = sorted(topology.hosts - set(named)) + refusals.append( + Refusal( + where, + "the answering hosts are not the listed ones" + + (f" (no answer for {', '.join(missing)})" if missing else ""), + ) + ) + if refusals: + raise Refused(refusals) + return out + + +async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: + """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs + no privilege. It prints the grants of every role the user holds, a role a session does + not have included: this first look names what is missing; the rows decide.""" + if missing := own_problems(await _show_grants(ch), topology, databases()): + own = await _own_user(ch) + return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] + return [] + + +async def own_refusals( + ch: Any, topology: Topology, names: Mapping[str, str] +) -> tuple[list[Refusal], HostGrants]: + """What attach's own user lacks to read what the checks read (without it `system.tables` + and `system.columns` would silently list less than there is), and every user's grants + per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several + hosts, that every host runs the reads as this user; then that user's grants as each host + keeps them; then the row policies, which would hide rows from its scan.""" + dbs = databases() + if lacking := await first_refusals(ch, topology): + return lacking, {} + own = await _own_user(ch) + try: + if topology.multi and (wrong := await _process_users(ch, topology, names, own)): + return wrong, {} + per_user, own_grants = await _host_grants(ch, topology, names, own) + out = [] + for host, grants in sorted(own_grants.items()): + # Each host keeps its own users: attach's must hold the same on every one, + # through its default roles alone, as its sessions have them. + if lacks := own_problems(grants, topology, dbs): + out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) + if not out: + out = await _row_policies(ch, topology, names, own, dbs) + except Exception as e: # only a refused read is judged here; the rest pass to the caller + if error_code(e) != _ACCESS_DENIED: + raise + reason = ( + f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " + "held only through a role that is not a default role does not reach its sessions" + ) + return [Refusal(f"user {own}", reason)], {} + return out, per_user + + +async def _row_policies( + ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] +) -> list[Refusal]: + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " + "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " + "WHERE database IN {dbs:Array(String)} OR database = 'system'", + {"dbs": dbs}, + ) + out = [] + for host, found in sorted(rows.items()): + for subject, why in row_policy_problems(found, own): + out.append(Refusal(subject, f"{why}{_on(host)}")) + return out + + +def _as_loom_writes(r: Mapping[str, Any]) -> bool: + """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in + either list could cover or exclude attach's user without naming it.""" + if r["short_name"] == POLICY_NAME: + return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] + return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] + + +def _applies(r: Mapping[str, Any], user: str) -> bool: + if r.get("apply_to_all"): + return user not in (r.get("apply_to_except") or []) + return user in (r.get("apply_to_list") or []) + + +def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: + """What hides rows from attach's own user, per row policy on a table of the configured + databases or a system table. Any policy on a system table refuses: the checks read them. + Once a table has a row policy, a user that no permissive policy covers reads none of its + rows, and a restrictive one filters them: either shortens the scan silently. Loom's own + pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` + letting every other user read every row) while it covers attach's user as written; every + other policy refuses.""" + tables: dict[str, list[Mapping[str, Any]]] = {} + for r in rows: + tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) + out = [] + for table, policies in sorted(tables.items()): + subject = f"row policy on {table}" + if table.startswith("system."): + # The checks read the system tables: a policy there hides what they read. Loom + # writes none. + out += [ + (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") + for p in policies + ] + continue + others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] + for p in policies: + name = str(p["short_name"]) + if name not in (POLICY_NAME, OTHERS_POLICY_NAME): + out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) + elif not _as_loom_writes(p): + out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) + elif name == POLICY_NAME and _applies(p, own): + out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) + if not any( + not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) + for p in others + ): + out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) + return out + + +async def _process_users( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> list[Refusal]: + """On several hosts: each host runs this read as attach's own user, exactly once.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, user FROM {source:processes} " + "WHERE initial_query_id = initialQueryID()", + ) + out = [] + for host, found in sorted(rows.items()): + users = [str(r["user"]) for r in found] + if users != [own]: + out.append( + Refusal( + f"host {host}", + f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " + "interserver secret, or the cluster's user to attach's", + ) + ) + return out + + +async def _host_grants( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> tuple[HostGrants, dict[str, list[Grant]]]: + """Per host, each user it defines and that user's grants, from `system.grants` and + `system.role_grants` on every topology (every granted role followed, default or not); + and per host the grants of `own` through its default roles alone.""" + users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") + # Every column: `access_object` and `is_wildcard` are read where the server has them. + grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") + roles = await per_host( + ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" + ) + every = { + host: { + str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found + } + for host, found in users.items() + } + own_grants = { + host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users + } + return every, own_grants + + +def _on(host: str) -> str: + return f" on host {host}" if host else "" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 4531ef81a7..06b8002c38 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -29,9 +29,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach, cell_attach_pg, cell_prepare -from server.cli.cell_ch import Refused -from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core import cell, cell_probe, cell_sync, cell_triggers, instances, pg from server.core import ch as ch_core +from server.core.cell_ch import Refused from server.core.schema import pg_migrate from server.router import app as router_app from server.router import directory @@ -895,9 +895,9 @@ async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) clock = [0.0] - real_budget = cell_scan.Budget + real_budget = cell_sync.Budget monkeypatch.setattr( - cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + cell_sync, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) ) # The budget runs out on the largest table, as it does on a real cell when that table's # read is what is too slow: every smaller one was read first. diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index c78bc40331..32a9560c6f 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -17,11 +17,13 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused -from server.cli.cell_scan import Budget, Table, TableList -from server.core import cell, ch_names, pg +from server.cli import cell_scan, cell_writers +from server.cli.cell_scan import Table, TableList +from server.core import cell, cell_ch, ch_names, pg +from server.core.cell_ch import Refused +from server.core.cell_sync import Budget from server.core.ch import get_client +from server.insights.store import cell_catalog from server.settings import settings from tests.integration._pg_database import own_database from tests.integration._resources import unique_org @@ -287,12 +289,12 @@ async def test_grants_are_read_as_the_server_keeps_them( await ch.execute(f"GRANT {role} TO {users['writer']}") await ch.execute(f"SET DEFAULT ROLE NONE TO {users['writer']}") topology = await cell_ch.read_topology(ch) - every, _ = await cell_writers._host_grants(ch, topology, {}, "") + every, _ = await cell_catalog._host_grants(ch, topology, {}, "") (grants,) = every.values() - assert f"SELECT ON {schema}.*" in cell_writers.own_problems( + assert f"SELECT ON {schema}.*" in cell_catalog.own_problems( grants[users["narrow"]], topology, [schema] ) - assert cell_writers.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] + assert cell_catalog.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] assert cell_writers.writer_reason(grants[users["writer"]], [schema]) == ( f"holds INSERT on {schema}.*" ) @@ -516,7 +518,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai created.remove(theirs) # Loom's own pair: the analyst's scope, and every other user reading every row, with # a scratch user standing in for the analyst. - monkeypatch.setattr(cell_writers, "ANALYST_USER", other) + monkeypatch.setattr(cell_catalog, "ANALYST_USER", other) for name, using, to in ( (POLICY_NAME, "org = ''", other), (OTHERS_POLICY_NAME, "1", f"ALL EXCEPT {other}"), @@ -527,7 +529,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai ) created.append(name) counted = await ch.query_one(f"SELECT count() AS n FROM {schema}.memory") - refusals = await cell_writers._row_policies(ch, topology, {}, own, cell_ch.databases()) + refusals = await cell_catalog._row_policies(ch, topology, {}, own, cell_ch.databases()) finally: for name in created: await ch.execute(f"DROP ROW POLICY IF EXISTS `{name}` ON {schema}.memory") diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index 37e80a7057..e2ab59c65b 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -251,7 +251,7 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( import httpx from server.cli import cell as cell_cli - from server.cli.cell_ch import Refused + from server.core.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index a289fb9b7f..6862f8ea0d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -18,9 +18,9 @@ from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal, Refused from server.cli.cell_scan import Table from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings from tests.unit.cell_control_owner import owner as _owner @@ -1443,8 +1443,8 @@ async def test_the_stray_read_names_a_refused_table_and_reads_on( ) -> None: from contextlib import asynccontextmanager - from server.cli import cell_ch, doctor_cell - from server.core import pg + from server.cli import doctor_cell + from server.core import cell_ch, pg async def topology(_ch: Any) -> Any: return cell_ch.Topology("", 0) diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py index 469ed16ea3..f63f195fda 100644 --- a/tests/unit/test_cell_attach_serving_identity.py +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -10,8 +10,8 @@ import pytest from server.cli import cell_attach, cell_control -from server.cli.cell_ch import Refused from server.core import pg +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FakeConn, _args diff --git a/tests/unit/test_cell_scan.py b/tests/unit/test_cell_scan.py index 56bf01a172..0366a44455 100644 --- a/tests/unit/test_cell_scan.py +++ b/tests/unit/test_cell_scan.py @@ -15,10 +15,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused, Topology -from server.cli.cell_scan import Budget, ScanResult, Table, TableList -from server.core import cell, pg +from server.cli import cell_scan +from server.cli.cell_scan import ScanResult, Table, TableList +from server.core import cell, cell_ch, pg +from server.core.cell_ch import Refused, Topology +from server.core.cell_sync import Budget +from server.insights.store import cell_catalog ONE = Topology("default", 1, (("localhost", 9000),)) TWO = Topology("main", 1, (("ch-0", 9000), ("ch-1", 9000))) @@ -178,7 +180,7 @@ async def test_each_host_answers_for_exactly_one_listed_name() -> None: [{"h": "os-0", "names": ["ch-0"]}, {"h": "os-1", "names": ["ch-1"]}], ) ) - assert await cell_writers.host_names(ch, TWO) == NAMES + assert await cell_catalog.host_names(ch, TWO) == NAMES # Every clusterAllReplicas read waits for every replica and reads its own host locally. reads = [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] assert "skip_unavailable_shards = 0, prefer_localhost_replica = 1" in reads[0] @@ -190,7 +192,7 @@ async def test_a_user_without_read_on_remote_is_named_before_any_cluster_read() ] ch = _granted(FakeCH(), narrow) with pytest.raises(Refused, match="attach's own user lacks READ ON REMOTE"): - await cell_writers.host_names(ch, TWO) + await cell_catalog.host_names(ch, TWO) assert not [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] @@ -208,7 +210,7 @@ async def test_a_host_without_one_configured_name_refuses( rows: list[dict[str, Any]], reason: str ) -> None: with pytest.raises(Refused, match=reason): - await cell_writers.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) + await cell_catalog.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) # -- the table list ---------------------------------------------------------------------- @@ -737,7 +739,7 @@ async def test_two_servers_under_one_host_name_refuse_before_any_read() -> None: same = Topology("main", 1, (("ch-0", 9000), ("ch-0", 9001))) ch = FakeCH() with pytest.raises(Refused, match="lists host ch-0 2 times"): - await cell_writers.host_names(ch, same) + await cell_catalog.host_names(ch, same) listed = await cell_scan.list_tables(ch, None, same, {}) assert [r.reason for r in listed.refusals] == [ "lists host ch-0 2 times (ports 9000, 9001): give each server its own host name" diff --git a/tests/unit/test_cell_scim_precondition.py b/tests/unit/test_cell_scim_precondition.py index 2c0bb0946f..0cf91e4792 100644 --- a/tests/unit/test_cell_scim_precondition.py +++ b/tests/unit/test_cell_scim_precondition.py @@ -11,8 +11,8 @@ from server.auth import authz, revocation, scim, subject_revocation from server.cli import cell_attach, cell_attach_beats -from server.cli.cell_ch import Refused from server.core import cell, ch +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FIELDS, FakeConn, _args, _beat, _value from tests.unit.test_scim_people import People diff --git a/tests/unit/test_cell_scim_precondition_failures.py b/tests/unit/test_cell_scim_precondition_failures.py index 11f46f0f60..ed175cc289 100644 --- a/tests/unit/test_cell_scim_precondition_failures.py +++ b/tests/unit/test_cell_scim_precondition_failures.py @@ -9,7 +9,7 @@ from server.auth import revocation from server.auth.revocation import scim_revoke as actual_scim_revoke -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from tests.unit.test_cell_attach_serving_identity import serving_run as serving_run from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py index 5465767496..353dc1e2cf 100644 --- a/tests/unit/test_cell_update_participants.py +++ b/tests/unit/test_cell_update_participants.py @@ -196,15 +196,15 @@ async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology events = [] topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(ch, top, names, sql, params): if "source:tables" in sql: @@ -235,7 +235,7 @@ async def query(sql, params): return [participant(1), participant(2), participant(1, "drained", version=2)] monkeypatch.setattr(module, "per_host", per_host) - monkeypatch.setattr(module.cell_scan, "_system", sync) + monkeypatch.setattr(module.cell_sync, "_system", sync) result = await module.census(SimpleNamespace(query_all=query)) assert len(result) == 2 and result[str(UUID(int=1))]["state"] == "drained" assert events == (["sync", "read"] if engine.startswith("Replicated") else ["read"]) @@ -251,7 +251,7 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology topology = Topology( "replicas", @@ -260,9 +260,9 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey ) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(*args): if problem == "unreachable": @@ -293,7 +293,7 @@ async def per_host(*args): monkeypatch.setattr(module, "per_host", per_host) monkeypatch.setattr( - module.cell_scan, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) + module.cell_sync, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) ) query = AsyncMock(side_effect=AssertionError("must not read partial census")) with pytest.raises((module.cell_control.Refused, OSError)): diff --git a/tests/unit/test_cell_update_scim_runtime.py b/tests/unit/test_cell_update_scim_runtime.py index 2cbb67094d..0fed38129c 100644 --- a/tests/unit/test_cell_update_scim_runtime.py +++ b/tests/unit/test_cell_update_scim_runtime.py @@ -12,9 +12,9 @@ from server.auth import scim from server.cli import cell_control, cell_update_participants, cell_update_probe, cell_update_run -from server.cli.cell_ch import Refused from server.cli.cell_update_pg import State from server.core import cell, pg_profile +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.cell_control_owner import owner from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_writers.py b/tests/unit/test_cell_writers.py index a3e85bca42..e1c84a457c 100644 --- a/tests/unit/test_cell_writers.py +++ b/tests/unit/test_cell_writers.py @@ -11,9 +11,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Topology -from server.cli.cell_writers import Allowed, Grant, holds, parse_show_grants, writer_reason +from server.cli import cell_scan, cell_writers +from server.cli.cell_writers import Allowed, writer_reason +from server.core import cell_ch +from server.core.cell_ch import Topology +from server.insights.store import cell_catalog +from server.insights.store.cell_catalog import Grant, holds, parse_show_grants from tests.unit.test_cell_scan import NAMES, ONE, TWO, FakeCH DBS = ["loom", "loom_telemetry"] @@ -69,7 +72,7 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: "REVOKE SELECT ON loom.secret FROM attach", "REVOKE GRANT OPTION FOR SELECT ON other.* FROM attach", ] - assert cell_writers.own_problems(parse_show_grants(lines), ONE, DBS) == [ + assert cell_catalog.own_problems(parse_show_grants(lines), ONE, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW TABLES ON side.*)", "SELECT ON loom.* (narrowed by REVOKE SELECT ON loom.secret)", ] @@ -90,8 +93,8 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: rows.append( {"user_name": "attach", "access_type": "SHOW", "database": "side", "is_partial_revoke": 1} ) - grants = cell_writers.resolve_grants("attach", rows, []) - assert cell_writers.own_problems(grants, TWO, DBS) == [ + grants = cell_catalog.resolve_grants("attach", rows, []) + assert cell_catalog.own_problems(grants, TWO, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW ON side.*)", "SHOW COLUMNS ON *.* (narrowed by REVOKE SHOW ON side.*)", ] @@ -101,10 +104,10 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: def test_attachs_own_user_needs_every_read_its_checks_make() -> None: full = [Grant("SELECT"), Grant("SHOW"), Grant("SHOW ACCESS"), Grant("SYSTEM")] - assert cell_writers.own_problems(full, ONE, DBS) == [] - assert cell_writers.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] + assert cell_catalog.own_problems(full, ONE, DBS) == [] + assert cell_catalog.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] narrow = [Grant("SELECT", "loom", "memory"), Grant("SELECT", "system"), Grant("SHOW")] - assert cell_writers.own_problems(narrow, Topology("", 0), DBS) == [ + assert cell_catalog.own_problems(narrow, Topology("", 0), DBS) == [ "SHOW USERS ON *.*", "SHOW ROLES ON *.*", "SHOW ROW POLICIES ON *.*", @@ -168,7 +171,7 @@ def test_a_grants_row_as_newer_servers_write_it() -> None: {"user_name": "u", "access_type": "WRITE", "access_object": "URL", "database": None}, {"user_name": "u", "access_type": "INSERT", "database": "lo", "is_wildcard": 1}, ] - assert cell_writers.resolve_grants("u", rows, []) == [ + assert cell_catalog.resolve_grants("u", rows, []) == [ Grant("REMOTE"), Grant("URL"), Grant("INSERT", "lo*"), @@ -190,9 +193,9 @@ def test_attachs_own_sessions_have_only_its_default_roles() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "spare", "granted_role_is_default": 0}, ] # fmt: skip - every = cell_writers.resolve_grants("u", grant_rows, role_rows) + every = cell_catalog.resolve_grants("u", grant_rows, role_rows) assert {g.access for g in every} == {"SHOW", "SELECT", "INSERT"} - session = cell_writers.resolve_grants("u", grant_rows, role_rows, default_only=True) + session = cell_catalog.resolve_grants("u", grant_rows, role_rows, default_only=True) assert {g.access for g in session} == {"SHOW", "SELECT"} @@ -203,7 +206,7 @@ def test_a_role_named_as_a_user_is_not_that_user() -> None: ] # Role `attach` holds role `w`; user `attach` holds neither. role_rows = [{"user_name": None, "role_name": "attach", "granted_role_name": "w"}] - assert cell_writers.resolve_grants("attach", grant_rows, role_rows) == [] + assert cell_catalog.resolve_grants("attach", grant_rows, role_rows) == [] def test_roles_are_followed_and_partial_revokes_ignored() -> None: @@ -224,7 +227,7 @@ def test_roles_are_followed_and_partial_revokes_ignored() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "outer"}, {"user_name": None, "role_name": "outer", "granted_role_name": "inner"}, ] - assert cell_writers.resolve_grants("u", grant_rows, role_rows) == [ + assert cell_catalog.resolve_grants("u", grant_rows, role_rows) == [ Grant("INSERT", "loom"), Grant("SELECT"), Grant("SELECT", "loom", "secret", revoked=True), @@ -470,7 +473,7 @@ def test_a_replica_keeper_lists_outside_the_cluster_is_extra() -> None: "replica_is_active": {"r0": 1}, }, ] - assert cell_writers.extra_replicas(rows) == {"loom.memory": {"dr"}} + assert cell_catalog.extra_replicas(rows) == {"loom.memory": {"dr"}} def _row(user: str, access: str, database: str | None = None) -> dict[str, Any]: @@ -775,7 +778,7 @@ def test_loom_s_own_row_policy_pair_lets_attach_read_every_row() -> None: _policy(POLICY_NAME, select_filter="org = 'a'", apply_to_list=["loom_analyst"]), _policy(OTHERS_POLICY_NAME, apply_to_all=1, apply_to_except=["loom_analyst"]), ] - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] def test_any_row_policy_on_a_system_table_refuses() -> None: @@ -783,7 +786,7 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: _policy("own", database="system", table="grants", apply_to_all=1), _policy("loom_analyst_others", database="system", table="tables", apply_to_all=1), ] - assert cell_writers.row_policy_problems(rows, "attach") == [ + assert cell_catalog.row_policy_problems(rows, "attach") == [ ("row policy on system.grants: own", "a row policy hides rows from the checks"), ( "row policy on system.tables: loom_analyst_others", @@ -839,13 +842,13 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: def test_a_row_policy_hiding_rows_from_attach_refuses( rows: list[dict[str, Any]], reason: str ) -> None: - found = cell_writers.row_policy_problems(rows, "attach") + found = cell_catalog.row_policy_problems(rows, "attach") assert any(reason in why for _, why in found), found async def test_the_row_policies_read_covers_the_system_tables() -> None: ch = _cluster() - await cell_writers.own_refusals(ch, TWO, NAMES) + await cell_catalog.own_refusals(ch, TWO, NAMES) (sql,) = [s for s in ch.sql("query_all") if "system.row_policies" in s] assert "database IN {dbs:Array(String)} OR database = 'system'" in sql @@ -924,14 +927,14 @@ def _looms_policies() -> list[dict[str, Any]]: def test_the_policies_loom_s_analyst_setup_writes_pass() -> None: rows = _looms_policies() assert {r["table"] for r in rows} >= {"memory", "analysis_canary"} - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] # Without every other user's policy, the canary would read no row for attach. alone = [ r for r in rows if not (r["table"] == "analysis_canary" and r["short_name"] == "loom_analyst_others") ] - assert cell_writers.row_policy_problems(alone, "attach") == [ + assert cell_catalog.row_policy_problems(alone, "attach") == [ ( "row policy on loom.analysis_canary", "no permissive policy `USING 1` covers attach: it reads no row", diff --git a/tests/unit/test_scim_review118_regressions.py b/tests/unit/test_scim_review118_regressions.py index 1df99b7173..182c13eaba 100644 --- a/tests/unit/test_scim_review118_regressions.py +++ b/tests/unit/test_scim_review118_regressions.py @@ -7,7 +7,7 @@ import pytest from server.auth import aliases, jit, login, oauth -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run from tests.unit.test_scim_people import people as people