diff --git a/docs/architecture/cells.md b/docs/architecture/cells.md index a78e60bce4..ff46557fb8 100644 --- a/docs/architecture/cells.md +++ b/docs/architecture/cells.md @@ -1716,7 +1716,7 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)). cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`; `server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories, objects and replicas outside the operator's allow lists); - `server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); + `server/core/cell_ch.py`: what both share (the cluster's hosts, the per-host reads); - `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and `schemas/migrations/094_cell_heartbeats.sql`: the cell's tables; - `server/auth/tenancy.py` (`ensure_held`): the tenancy check and the placement call, diff --git a/server/cli/cell_attach.py b/server/cli/cell_attach.py index b2e6b25059..241f9143b5 100644 --- a/server/cli/cell_attach.py +++ b/server/cli/cell_attach.py @@ -46,8 +46,9 @@ from server.cli import cell_attach_pg as apg from server.cli import cell_control, cell_scan, cell_scim from server.cli._ch import writes -from server.cli.cell_ch import Refusal, Refused, read_topology -from server.core import cell, cell_triggers +from server.core import cell, cell_sync, cell_triggers +from server.core.cell_ch import Refusal, Refused, read_topology +from server.insights.store import cell_catalog from server.settings import settings _WAIT_S = 600 # not a setting: a flag's default @@ -324,7 +325,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]: topology = await read_topology(self.ch) if refusals := topology.refusals(): return refusals, topology, {} - names = await cell_writers.host_names(self.ch, topology) if topology.multi else {} + names = await cell_catalog.host_names(self.ch, topology) if topology.multi else {} allowed = cell_writers.Allowed( users=frozenset(self.args.ch_allow_user), user_directories=frozenset(self.args.ch_allow_user_directory), @@ -406,7 +407,7 @@ async def begin(self) -> None: # -- the scans --------------------------------------------------------------------- async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult: - budget = cell_scan.Budget(self.args.scan_budget) + budget = cell_sync.Budget(self.args.scan_budget) current, nxt = self.own_fp return await cell_scan.scan( self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt) diff --git a/server/cli/cell_attach_beats.py b/server/cli/cell_attach_beats.py index 57cc73bfd1..539e2982f3 100644 --- a/server/cli/cell_attach_beats.py +++ b/server/cli/cell_attach_beats.py @@ -26,8 +26,8 @@ from typing import Any, Literal from server.cli.cell_attach_pg import Wait -from server.cli.cell_ch import Refusal, Refused from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused StoppedUnset = Literal["refuse", "wait"] diff --git a/server/cli/cell_attach_pg.py b/server/cli/cell_attach_pg.py index 89717d5699..0ca96ce4c0 100644 --- a/server/cli/cell_attach_pg.py +++ b/server/cli/cell_attach_pg.py @@ -18,9 +18,9 @@ from functools import partial from typing import TYPE_CHECKING, Any -from server.cli.cell_ch import Refusal, Refused from server.core import cell, cell_triggers, pg from server.core.cell import ATTACH_LOCK_KEY +from server.core.cell_ch import Refusal, Refused if TYPE_CHECKING: from server.cli.cell_control import Owner diff --git a/server/cli/cell_control.py b/server/cli/cell_control.py index 0fbaade144..53d65b7838 100644 --- a/server/cli/cell_control.py +++ b/server/cli/cell_control.py @@ -13,9 +13,9 @@ from uuid import UUID from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal -from server.cli.cell_ch import Refused as Refused from server.core import pg, retirement +from server.core.cell_ch import Refusal +from server.core.cell_ch import Refused as Refused from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S from server.settings import settings diff --git a/server/cli/cell_place.py b/server/cli/cell_place.py index b564ee0cff..321345618b 100644 --- a/server/cli/cell_place.py +++ b/server/cli/cell_place.py @@ -14,9 +14,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach_pg as apg from server.cli._ch import run_connected, writes -from server.cli.cell_ch import Refused from server.cli.cell_control import Owner from server.core import cell, pg +from server.core.cell_ch import Refused from server.router.directory import PLACE_MAX_S from server.settings import settings diff --git a/server/cli/cell_scan.py b/server/cli/cell_scan.py index f66435373f..1f292031b7 100644 --- a/server/cli/cell_scan.py +++ b/server/cli/cell_scan.py @@ -24,14 +24,13 @@ import asyncio import json import re -import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any -from clickhouse_connect.driver.exceptions import DatabaseError - -from server.cli.cell_ch import ( +from server.core import cell_sync, ch_errors, ch_names, pg +from server.core.cell import ORG_FREE, named_org, org_name_refusal +from server.core.cell_ch import ( TAG, WHOLE, WHOLE_SQL, @@ -42,14 +41,12 @@ per_host, quote, ) -from server.cli.cell_writers import own_refusals -from server.core import ch_errors, ch_names, pg -from server.core.cell import ORG_FREE, named_org, org_name_refusal from server.core.cell_triggers import ORG_TABLES from server.core.ch import query_tag from server.core.ch_errors import error_code, error_line from server.erasure.store.census import classified_tables from server.erasure.store.steps_pg import PG_CENSUS, PG_NO_MEMORY_CONTENT +from server.insights.store.cell_catalog import own_refusals # Tables whose orgs are never placed: rows written before any check (a refused or # unauthenticated request's log row keeps the org it named), and tables a view fills from @@ -91,38 +88,11 @@ # How long past the budget a Postgres try may take before this side gives it up, so the # server's own statement timeout fires first when the server answers. _PG_MARGIN_S = 5.0 # not a setting: a margin over --scan-budget -_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED _UNKNOWN_TABLE_CODES = frozenset({60, 81}) # UNKNOWN_TABLE, UNKNOWN_DATABASE _sleep = asyncio.sleep # the seam the retry tests replace -class Budget: - """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" - - def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: - self.seconds = float(seconds) - self._clock = clock - self._start = clock() - - def elapsed(self) -> float: - return self._clock() - self._start - - def left(self) -> float: - return self.seconds - self.elapsed() - - def require(self, subject: str) -> float: - """The seconds left, or Refused naming `subject` once none are.""" - left = self.left() - if left <= 0: - raise Refused([_out_of_budget(subject, self.seconds)]) - return left - - -def _out_of_budget(subject: str, seconds: float) -> Refusal: - return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") - - def pg_quote(name: str) -> str: return '"' + name.replace('"', '""') + '"' @@ -383,7 +353,7 @@ async def list_tables( ) -> TableList: """The scan's table list in both stores, with every refusal it found, attach's own ClickHouse grants checked first; raises `Refused` when a host answers outside the - cluster's names. `names` is `cell_writers.host_names()` on a cluster of several hosts, + cluster's names. `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty.""" with query_tag(TAG): return await _list_tables(ch, pg_conn, topology, names) @@ -501,7 +471,7 @@ def pg_skip_scan_sql(table: str) -> str: return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}") -async def pg_orgs(table: str, budget: Budget) -> list[str]: +async def pg_orgs(table: str, budget: cell_sync.Budget) -> list[str]: subject = f"postgres {table}" while True: left = budget.require(subject) @@ -529,7 +499,7 @@ async def pg_orgs(table: str, budget: Budget) -> list[str]: # budget ran out only when there is none left, since the timeout is what was left. timed_out = getattr(failure, "sqlstate", None) == "57014" and budget.left() <= 1 if bound.expired() or timed_out: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None # A connect that finds Postgres down comes as PostgresUnavailableError. if not (pg.is_outage(failure) or isinstance(failure, pg.PostgresUnavailableError)): raise Refused([Refusal(subject, f"the scan failed: {failure}")]) from failure @@ -545,63 +515,6 @@ def _ch_transient(e: BaseException) -> bool: ) -def _on_cluster(topology: Topology) -> str: - return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" - - -async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: - """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries - `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must - answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a - profile's `never_throw` hides none). One that does not finish in time refuses as the - budget when the budget is spent, else with the server's error and the hosts it did not - finish on: a replica that cannot sync.""" - seconds = max(1, int(budget.require(subject))) - # `receive_timeout` bounds SYNC REPLICA's own wait. - bounds = {"max_execution_time": seconds, "receive_timeout": seconds} - what = sql.split(" ON CLUSTER ")[0].split(" `")[0] - try: - if not topology.multi: - await ch.execute(sql, settings=bounds) - return - body = await ch.raw_bytes( - sql, - settings={ - **bounds, - "distributed_ddl_output_mode": "throw", - "distributed_ddl_task_timeout": seconds, - "default_format": "JSONEachRow", - }, - ) - rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") - except Exception as e: # only a timeout is judged here; the rest pass to the caller - if error_code(e) not in _TIMEOUT_CODES: - raise - # The statement was given the budget's whole seconds, so up to one is left over. - if budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} - unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" - raise Refused( - [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] - ) from e - failed = [r for r in rows if int(r["status"]) != 0] - if failed: - raise Refused( - [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] - ) - answered = {str(r.get("host")) for r in rows} - if answered != topology.hosts: - missing = ", ".join(sorted(topology.hosts - answered)) or "none" - raise Refused( - [ - Refusal( - subject, f"the hosts that answered are not the listed ones (missing: {missing})" - ) - ] - ) - - def ch_scan_sql(table: Table, topology: Topology) -> str: """The distinct orgs of `table` as hex (bytes: an org that is not UTF-8 is judged, never decoded into another name); on several hosts, first a row naming the configured host @@ -616,36 +529,6 @@ def ch_scan_sql(table: Table, topology: Topology) -> str: return f"{host} UNION ALL {orgs} FORMAT JSONEachRow" -class BodyError(DatabaseError): - """An error ClickHouse wrote into a response body after it began streaming rows: a - server error as the driver raises one, its text keeping the server's `Code: .`, so it - is judged transient or not, and by code, as any other. `rows` are the rows before it.""" - - def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: - super().__init__(text) - self.rows = list(rows or []) - - -def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: - """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that - is not one (an `{"exception": …}` row, or the text tail a server appends once a - statement fails after its first block), with the rest of the body as its text.""" - lines = list(lines) - rows: list[dict[str, Any]] = [] - for i, line in enumerate(lines): - if not line.strip(): - continue - try: - row = json.loads(line) - except ValueError: - row = None - if not isinstance(row, dict) or key not in row: - exception = row.get("exception") if isinstance(row, dict) else None - raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) - rows.append(row) - return rows - - async def _read_scan( ch: Any, sql: str, params: dict[str, Any], extra: dict[str, Any] ) -> list[dict[str, Any]]: @@ -653,23 +536,25 @@ async def _read_scan( async with ch.raw_stream(sql, params, settings=extra) as stream: async for chunk in stream: chunks.append(chunk) - return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") + return cell_sync._body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k") -async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]: +async def ch_orgs( + ch: Any, table: Table, topology: Topology, budget: cell_sync.Budget +) -> list[bytes | None]: subject = f"clickhouse {table.canonical}" while True: budget.require(subject) try: # Rows buffered on any replica land before the table is read. - flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{_on_cluster(topology)}" - await _system(ch, flush, topology, budget, subject) + flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{cell_sync._on_cluster(topology)}" + await cell_sync._system(ch, flush, topology, budget, subject) if table.engine.startswith("Replicated"): # Every replica pulls every part acknowledged before now, so the read # through any one of them sees them all. - await _system( + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{_on_cluster(topology)} {table.sql} LIGHTWEIGHT", + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} {table.sql} LIGHTWEIGHT", topology, budget, subject, @@ -695,9 +580,9 @@ async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> if r["k"] == "o" ] # A timeout is the budget only when none is left: a profile's estimate throws early. - if error_code(failure) in _TIMEOUT_CODES and budget.left() <= 1: - raise Refused([_out_of_budget(subject, budget.seconds)]) from None - if error_code(failure) in _TIMEOUT_CODES or not _ch_transient(failure): + if error_code(failure) in cell_sync._TIMEOUT_CODES and budget.left() <= 1: + raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None + if error_code(failure) in cell_sync._TIMEOUT_CODES or not _ch_transient(failure): raise Refused( [Refusal(subject, f"the scan failed: {error_line(failure)}")] ) from failure @@ -722,7 +607,7 @@ async def scan( ch: Any, tables: TableList, topology: Topology, - budget: Budget, + budget: cell_sync.Budget, configured: Iterable[str] = (), ) -> ScanResult: """The distinct orgs of every table of `tables` and the `configured` orgs; Refused, with diff --git a/server/cli/cell_scim.py b/server/cli/cell_scim.py index aa0573c347..230b378939 100644 --- a/server/cli/cell_scim.py +++ b/server/cli/cell_scim.py @@ -14,8 +14,8 @@ from server.auth import claims, revocation, scim, scim_people, subject_revocation from server.auth.scim_routes import scim_settings_problem from server.cli import cell_control -from server.cli.cell_ch import WHOLE_SQL, Refusal, Refused from server.core import cell, ch +from server.core.cell_ch import WHOLE_SQL, Refusal, Refused from server.core.datetimes import as_aware_utc from server.settings import settings diff --git a/server/cli/cell_update_participants.py b/server/cli/cell_update_participants.py index 26a3fa8c73..b948d47e8e 100644 --- a/server/cli/cell_update_participants.py +++ b/server/cli/cell_update_participants.py @@ -9,9 +9,9 @@ from typing import Any, cast from uuid import UUID -from server.cli import cell_control, cell_scan, cell_writers -from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology -from server.core import cell, ch_names, instances +from server.cli import cell_control +from server.core import cell, cell_sync, ch_names, instances +from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology from server.core.ch import query_tag from server.core.provider_process import ( certified, @@ -19,6 +19,7 @@ participant_tag, proves_stopped, ) +from server.insights.store import cell_catalog from server.insights.store import provider_participants as participants @@ -88,8 +89,8 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: raise cell_control._refused("participant census cannot resolve the configured cluster") if problems := topology.refusals(): raise Refused(problems) - names = await cell_writers.host_names(ch, topology) if topology.multi else {} - if problems := (await cell_writers.own_refusals(ch, topology, names))[0]: + names = await cell_catalog.host_names(ch, topology) if topology.multi else {} + if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]: raise Refused(problems) database = ch_names.database() tables = await per_host( @@ -122,14 +123,14 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]: ), {"db": database}, ) - if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas( + if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas( row for value in rows.values() for row in value ): raise cell_control._refused("participant replica set includes an unverified replica") - budget = cell_scan.Budget(cell_control.TRANSACTION_S) - await cell_scan._system( + budget = cell_sync.Budget(cell_control.TRANSACTION_S) + await cell_sync._system( ch, - f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} " + f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} " f"{quote(database)}.`_provider_participants` LIGHTWEIGHT", topology, budget, diff --git a/server/cli/cell_writers.py b/server/cli/cell_writers.py index 0aa0c8e0b8..2d7e6345af 100644 --- a/server/cli/cell_writers.py +++ b/server/cli/cell_writers.py @@ -31,37 +31,15 @@ from dataclasses import dataclass from typing import Any -from server.cli.cell_ch import ( - CLUSTER_SQL, - TAG, - WHOLE_SQL, +from server.core.cell_ch import ( WRITERS_TAG, Refusal, - Refused, Topology, databases, per_host, ) -from server.core import ch_names from server.core.ch import query_tag -from server.core.ch_errors import error_code, error_line -from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME - -_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED - -# Each privilege attach needs and the grants that imply it, closest first. -_IMPLIED_BY: dict[str, tuple[str, ...]] = { - "SELECT": ("ALL",), - "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), - "SHOW TABLES": ("SHOW", "ALL"), - "SHOW COLUMNS": ("SHOW", "ALL"), - "REMOTE": ("SOURCES", "ALL"), - "CLUSTER": ("ALL",), - "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), - "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), -} +from server.insights.store import cell_catalog # Grants that write the configured databases or hand that right on: a user holding one on # them, or globally, must be one the operator names. @@ -163,235 +141,34 @@ class Allowed: replicas: frozenset[str] = frozenset() -@dataclass(frozen=True) -class Grant: - """One privilege a user holds: `database` None for a global grant, `table` None for the - whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial - revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" - - access: str - database: str | None = None - table: str | None = None - grant_option: bool = False - revoked: bool = False - - def covers(self, database: str | None) -> bool: - """Whether this grant reaches all of `database` (None: every database).""" - if self.table is not None: - return False - return self.database is None or (database is not None and _db_match(self, database)) - - def touches(self, database: str) -> bool: - """Whether this grant reaches any table of `database`.""" - return self.database is None or _db_match(self, database) - - -def _db_match(grant: Grant, database: str) -> bool: - assert grant.database is not None - if grant.database.endswith("*"): - return database.startswith(grant.database[:-1]) - return grant.database == database - - -def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: - """Whether `grants` give `access` on all of `database` (None: globally).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) - - -def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: - """The partial revokes in `grants` of `access`, or of a grant implying it, on any part - of `database` (None: anywhere).""" - wanted = {access, *_IMPLIED_BY.get(access, ())} - return [ - g - for g in grants - if g.revoked - and g.access in wanted - and (database is None or g.database is None or _db_match(g, database)) - ] - - -# -- reading grants ---------------------------------------------------------------------- - -_GRANT = re.compile( - r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " - r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" -) -_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") - - -def _split_privileges(text: str) -> list[str]: - """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" - out, depth, start = [], 0, 0 - for i, ch in enumerate(text): - depth += {"(": 1, ")": -1}.get(ch, 0) - if ch == "," and depth == 0: - out.append(text[start:i]) - start = i + 1 - out.append(text[start:]) - return [p.split("(")[0].strip() for p in out if p.strip()] - - -def _target(on: str) -> tuple[str | None, str | None] | str: - """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a - `READ` grant names (`REMOTE`) as itself.""" - if on == "*": - return (None, None) - parts = [a or b for a, b in _IDENT.findall(on)] - if len(parts) != 2: - return on - db, table = parts - return (None if db == "*" else db, None if table == "*" else table) - - -def parse_show_grants(lines: Iterable[str]) -> list[Grant]: - """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its - roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the - revokes of a grant option alone are left out.""" - out = [] - for line in lines: - m = _GRANT.match(line.strip()) - if m is None or m["privs"].startswith("GRANT OPTION FOR "): - continue - target = _target(m["on"]) - option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" - for access in _split_privileges(m["privs"]): - if isinstance(target, str): - # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it - # as written. - if access in ("READ", "WRITE"): - out.append(Grant(target, grant_option=option, revoked=revoked)) - continue - out.append(Grant(access, target[0], target[1], option, revoked)) - return out - - -def resolve_grants( - user: str, - grant_rows: Iterable[Mapping[str, Any]], - role_rows: Iterable[Mapping[str, Any]], - *, - default_only: bool = False, -) -> list[Grant]: - """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every - role it holds followed (default or not, since it can set any). `default_only` follows - only the user's default roles, the ones a new session has (attach's own reads run in - one), and every role those hold: `granted_role_is_default` marks the user's own grants, - while a role's grant of another always passes it on.""" - # A user and a role may share a name: each holder is keyed by its kind. - granted: dict[tuple[str, str], set[str]] = {} - for r in role_rows: - holder = ( - ("user", str(r["user_name"])) - if r.get("user_name") - else ("role", str(r.get("role_name"))) - ) - if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: - continue - granted.setdefault(holder, set()).add(str(r["granted_role_name"])) - roles: set[str] = set() - todo = list(granted.get(("user", user), ())) - while todo: - role = todo.pop() - if role not in roles: - roles.add(role) - todo.extend(granted.get(("role", role), ())) - out = [] - for r in grant_rows: - if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): - out.append(_row_grant(r)) - return out - - -def _row_grant(r: Mapping[str, Any]) -> Grant: - """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object - `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database - as a prefix.""" - access = str(r["access_type"]) - if access in ("READ", "WRITE") and r.get("access_object"): - access = str(r["access_object"]) - database = None if r.get("database") is None else str(r["database"]) - if database is not None and r.get("is_wildcard") and not database.endswith("*"): - database += "*" - return Grant( - access, - database, - None if r.get("table") is None else str(r["table"]), - bool(r.get("grant_option")), - bool(r.get("is_partial_revoke")), - ) - - # -- judging ----------------------------------------------------------------------------- -def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: - """What attach's own user lacks to read what the check and the scan read.""" - needs: list[tuple[str, str | None]] = [ - ("SELECT", "system"), - ("SHOW USERS", None), - ("SHOW ROLES", None), - ("SHOW ROW POLICIES", None), - ("SHOW TABLES", None), - ("SHOW COLUMNS", None), - ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), - ] - for db in dbs: - needs.append(("SELECT", db)) - if topology.hosts: - needs.append(("SYSTEM SYNC REPLICA", db)) - if topology.multi: - needs += [("REMOTE", None), ("CLUSTER", None)] - out = [] - for access, scope in needs: - if not holds(grants, access, scope): - out.append(_label(access, scope)) - elif revokes := narrowing(grants, access, scope): - # A revoke hides part of what the checks read, which reads short silently. - out.append( - f"{_label(access, scope)} (narrowed by REVOKE " - + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) - + ")" - ) - return out - - -def _label(access: str, db: str | None) -> str: - if access == "REMOTE": - return "READ ON REMOTE" - return f"{access} ON {db}.*" if db else f"{access} ON *.*" - - -def writer_reason(grants: Iterable[Grant], dbs: list[str]) -> str: +def writer_reason(grants: Iterable[cell_catalog.Grant], dbs: list[str]) -> str: """Why a user with `grants` can write the configured databases ('' when it cannot): a grant that writes them from anywhere (`_ANYWHERE_ACCESS`), a writing grant on them or globally, else any grant there it can hand on.""" grants = [g for g in grants if not g.revoked] # a revoke only narrows: ignored for g in grants: if g.access in _ANYWHERE_ACCESS: - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" reaching = [g for g in grants if any(g.touches(db) for db in dbs)] for g in reaching: if g.access in _WRITER_ACCESS or g.access.startswith("ALTER "): - return f"holds {g.access} on {_scope(g)}" + return f"holds {g.access} on {cell_catalog._scope(g)}" for g in reaching: if g.grant_option: - return f"holds {g.access} WITH GRANT OPTION on {_scope(g)}" + return f"holds {g.access} WITH GRANT OPTION on {cell_catalog._scope(g)}" # A row policy created later hides rows from the next scan or check, as a write made # later adds them: on the configured databases or the system tables the checks read. for g in grants: if g.access in _POLICY_ACCESS and (g.touches("system") or any(g.touches(db) for db in dbs)): - return f"holds {g.access} on {_scope(g)}, which can hide rows from the scan" + return ( + f"holds {g.access} on {cell_catalog._scope(g)}, which can hide rows from the scan" + ) return "" -def _scope(g: Grant) -> str: - if g.database is None: - return "*.*" - return f"{g.database}.{g.table or '*'}" - - _TO = re.compile(r"\bTO\s+(?:`((?:[^`\\]|\\.)*)`|(\w+))\s*\.", re.I) _NAME = r"(?:`(?:[^`\\]|\\.)*`|[A-Za-z_]\w*)" # `CREATE … VIEW `: a view's `TO` target follows it, and its name may hold anything. @@ -646,280 +423,19 @@ def _inside_problem( return f"a table of engine {engine}, which reads outside the configured databases" -def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: - """Per replicated table (`db.table`), the replicas Keeper lists that no host of the - cluster is (`replica_is_active` keys against every host's `replica_name`, by - `zookeeper_path`).""" - ours: dict[str, set[str]] = {} - listed: dict[str, set[str]] = {} - names: dict[str, str] = {} - for r in rows: - path = str(r["zookeeper_path"]) - ours.setdefault(path, set()).add(str(r["replica_name"])) - listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) - names.setdefault(path, f"{r['database']}.{r['table']}") - out = {} - for path, keys in listed.items(): - if extra := keys - ours[path]: - out[names[path]] = extra - return out - - # -- the check --------------------------------------------------------------------------- -async def _own_user(ch: Any) -> str: - row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") - return str((row or {}).get("u") or "") - - -async def _show_grants(ch: Any) -> list[Grant]: - """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no - privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" - # Pinned: a profile's `break` could end the list before its REVOKE lines. - rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") - return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) - - -# Per host, each user it defines and that user's grants. -HostGrants = dict[str, dict[str, list[Grant]]] - - -async def host_names(ch: Any, topology: Topology) -> dict[str, str]: - """Each answering host's OS hostname (an opaque group key) and the one configured name - it answers for in the cluster; Refused unless there is one group per listed host, each - with exactly one name, the names together the listing. The topology's own refusals (two - servers under one host name, several shards) come first, then what attach's own user - lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read - through `clusterAllReplicas`.""" - if refusals := topology.refusals(): - raise Refused(refusals) - with query_tag(TAG): - if lacking := await first_refusals(ch, topology): - raise Refused(lacking) - rows = await ch.query_all( - "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " - "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " - f"GROUP BY h {CLUSTER_SQL}", - {"c": topology.cluster}, - ) - where = f"cluster {topology.cluster}" - refusals = [] - out: dict[str, str] = {} - for r in rows: - names = sorted(str(n) for n in r["names"]) - if not names: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host's address in {where} does not resolve to a local interface", - ) - ) - elif len(names) > 1: - refusals.append( - Refusal( - f"host {r['h']}", - f"this host answers for several entries in {where} ({', '.join(names)}): " - "two local entries, or two hosts with one OS hostname", - ) - ) - else: - out[str(r["h"])] = names[0] - named = sorted(out.values()) - if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): - missing = sorted(topology.hosts - set(named)) - refusals.append( - Refusal( - where, - "the answering hosts are not the listed ones" - + (f" (no answer for {', '.join(missing)})" if missing else ""), - ) - ) - if refusals: - raise Refused(refusals) - return out - - -async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: - """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs - no privilege. It prints the grants of every role the user holds, a role a session does - not have included: this first look names what is missing; the rows decide.""" - if missing := own_problems(await _show_grants(ch), topology, databases()): - own = await _own_user(ch) - return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] - return [] - - -async def own_refusals( - ch: Any, topology: Topology, names: Mapping[str, str] -) -> tuple[list[Refusal], HostGrants]: - """What attach's own user lacks to read what the checks read (without it `system.tables` - and `system.columns` would silently list less than there is), and every user's grants - per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several - hosts, that every host runs the reads as this user; then that user's grants as each host - keeps them; then the row policies, which would hide rows from its scan.""" - dbs = databases() - if lacking := await first_refusals(ch, topology): - return lacking, {} - own = await _own_user(ch) - try: - if topology.multi and (wrong := await _process_users(ch, topology, names, own)): - return wrong, {} - per_user, own_grants = await _host_grants(ch, topology, names, own) - out = [] - for host, grants in sorted(own_grants.items()): - # Each host keeps its own users: attach's must hold the same on every one, - # through its default roles alone, as its sessions have them. - if lacks := own_problems(grants, topology, dbs): - out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) - if not out: - out = await _row_policies(ch, topology, names, own, dbs) - except Exception as e: # only a refused read is judged here; the rest pass to the caller - if error_code(e) != _ACCESS_DENIED: - raise - reason = ( - f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " - "held only through a role that is not a default role does not reach its sessions" - ) - return [Refusal(f"user {own}", reason)], {} - return out, per_user - - -async def _row_policies( - ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] -) -> list[Refusal]: - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " - "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " - "WHERE database IN {dbs:Array(String)} OR database = 'system'", - {"dbs": dbs}, - ) - out = [] - for host, found in sorted(rows.items()): - for subject, why in row_policy_problems(found, own): - out.append(Refusal(subject, f"{why}{_on(host)}")) - return out - - -def _as_loom_writes(r: Mapping[str, Any]) -> bool: - """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in - either list could cover or exclude attach's user without naming it.""" - if r["short_name"] == POLICY_NAME: - return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] - return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] - - -def _applies(r: Mapping[str, Any], user: str) -> bool: - if r.get("apply_to_all"): - return user not in (r.get("apply_to_except") or []) - return user in (r.get("apply_to_list") or []) - - -def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: - """What hides rows from attach's own user, per row policy on a table of the configured - databases or a system table. Any policy on a system table refuses: the checks read them. - Once a table has a row policy, a user that no permissive policy covers reads none of its - rows, and a restrictive one filters them: either shortens the scan silently. Loom's own - pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` - letting every other user read every row) while it covers attach's user as written; every - other policy refuses.""" - tables: dict[str, list[Mapping[str, Any]]] = {} - for r in rows: - tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) - out = [] - for table, policies in sorted(tables.items()): - subject = f"row policy on {table}" - if table.startswith("system."): - # The checks read the system tables: a policy there hides what they read. Loom - # writes none. - out += [ - (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") - for p in policies - ] - continue - others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] - for p in policies: - name = str(p["short_name"]) - if name not in (POLICY_NAME, OTHERS_POLICY_NAME): - out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) - elif not _as_loom_writes(p): - out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) - elif name == POLICY_NAME and _applies(p, own): - out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) - if not any( - not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) - for p in others - ): - out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) - return out - - -async def _process_users( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> list[Refusal]: - """On several hosts: each host runs this read as attach's own user, exactly once.""" - rows = await per_host( - ch, - topology, - names, - "SELECT hostName() AS h, user FROM {source:processes} " - "WHERE initial_query_id = initialQueryID()", - ) - out = [] - for host, found in sorted(rows.items()): - users = [str(r["user"]) for r in found] - if users != [own]: - out.append( - Refusal( - f"host {host}", - f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " - "interserver secret, or the cluster's user to attach's", - ) - ) - return out - - -async def _host_grants( - ch: Any, topology: Topology, names: Mapping[str, str], own: str -) -> tuple[HostGrants, dict[str, list[Grant]]]: - """Per host, each user it defines and that user's grants, from `system.grants` and - `system.role_grants` on every topology (every granted role followed, default or not); - and per host the grants of `own` through its default roles alone.""" - users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") - # Every column: `access_object` and `is_wildcard` are read where the server has them. - grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") - roles = await per_host( - ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" - ) - every = { - host: { - str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found - } - for host, found in users.items() - } - own_grants = { - host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users - } - return every, own_grants - - -def _on(host: str) -> str: - return f" on host {host}" if host else "" - - async def check_writers( ch: Any, topology: Topology, names: Mapping[str, str], allowed: Allowed ) -> list[Refusal]: """Every ClickHouse writer of the configured databases the operator has not named. - `names` is `host_names()` on a cluster of several hosts, else empty. Raises + `names` is `cell_catalog.host_names()` on a cluster of several hosts, else empty. Raises `cell_ch.Refused` when a host answers outside the cluster's names; every other refusal is returned.""" dbs = databases() with query_tag(WRITERS_TAG): - lacking, per_user = await own_refusals(ch, topology, names) + lacking, per_user = await cell_catalog.own_refusals(ch, topology, names) if lacking: return lacking refusals: list[Refusal] = [] @@ -927,7 +443,10 @@ async def check_writers( for user, grants in sorted(users.items()): if user not in allowed.users and (why := writer_reason(grants, dbs)): refusals.append( - Refusal(f"user {user}", f"{why}{_on(host)}: name it with --ch-allow-user") + Refusal( + f"user {user}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-user", + ) ) refusals += await _directories(ch, topology, names, allowed) refusals += await _objects(ch, topology, names, allowed, dbs) @@ -949,7 +468,8 @@ async def _directories( out.append( Refusal( f"user directory {name}", - f"its type, {kind}, defines users this check cannot read{_on(host)}: " + f"its type, {kind}, defines users this check cannot read" + f"{cell_catalog._on(host)}: " "name it with --ch-allow-user-directory", ) ) @@ -980,7 +500,10 @@ async def _objects( for key, why in sorted(object_problems(found, dbs, functions, topology.cluster).items()): if key not in allowed.objects: out.append( - Refusal(f"object {key}", f"{why}{_on(host)}: name it with --ch-allow-object") + Refusal( + f"object {key}", + f"{why}{cell_catalog._on(host)}: name it with --ch-allow-object", + ) ) return out @@ -998,7 +521,7 @@ async def _replicas( ) every = [r for found in rows.values() for r in found] out = [] - for table, extra in sorted(extra_replicas(every).items()): + for table, extra in sorted(cell_catalog.extra_replicas(every).items()): for replica in sorted(extra): if f"{table}:{replica}" not in allowed.replicas: out.append( diff --git a/server/cli/doctor_cell.py b/server/cli/doctor_cell.py index 292f03f374..194964832c 100644 --- a/server/cli/doctor_cell.py +++ b/server/cli/doctor_cell.py @@ -20,6 +20,7 @@ from server.cli._ch import reads from server.cli.doctor import FAIL, PASS, WARN +from server.core import cell_sync # A doctor run never hangs on a cell's stores: each statement, and each session's connect # and reads together. @@ -263,13 +264,13 @@ async def _table_sizes(ch: Any, tables: list[Any]) -> dict[str, int]: async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: from server.cli import cell_scan - from server.cli.cell_ch import Refused, databases, read_topology from server.core import pg + from server.core.cell_ch import Refused, databases, read_topology from server.core.cell_triggers import ORG_TABLES topology = await read_topology(ch) if topology.multi: - from server.cli.cell_writers import host_names + from server.insights.store.cell_catalog import host_names names = await host_names(ch, topology) else: @@ -289,7 +290,7 @@ async def _stray_reads(ch: Any, budget_s: float) -> list[tuple[str, str, str]]: work: list[tuple[int, str, Any]] = [(int(r["n"]), "postgres", str(r["name"])) for r in pg_rows] work += [(sizes.get(t.canonical, 0), "clickhouse", t) for t in ch_tables] work.sort(key=lambda w: (w[0], str(w[2]))) - budget = cell_scan.Budget(budget_s) + budget = cell_sync.Budget(budget_s) stray: dict[str, list[str]] = {} unchecked: list[str] = [] refused: list[str] = [] diff --git a/server/cli/cell_ch.py b/server/core/cell_ch.py similarity index 99% rename from server/cli/cell_ch.py rename to server/core/cell_ch.py index 3da3766379..6262a23176 100644 --- a/server/cli/cell_ch.py +++ b/server/core/cell_ch.py @@ -157,7 +157,7 @@ async def per_host( """The rows of `sql`, which selects `hostName() AS h` from `{source:}`, grouped by the configured name of the host each came from: system table `
` itself on one host (grouped under ''), through `clusterAllReplicas` on several (grouped by `names`, - from `cell_writers.host_names()`).""" + from `cell_catalog.host_names()`).""" m = re.search(r"\{source:(\w+)\}", sql) assert m, sql table = f"system.{m.group(1)}" diff --git a/server/core/cell_sync.py b/server/core/cell_sync.py new file mode 100644 index 0000000000..e723a41784 --- /dev/null +++ b/server/core/cell_sync.py @@ -0,0 +1,128 @@ +"""Complete replica synchronization with the caller's existing scan budget.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable, Iterable +from typing import Any + +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core.cell_ch import Refusal, Refused, Topology, quote +from server.core.ch_errors import error_code, error_line + +_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED + + +class Budget: + """`--scan-budget`: the seconds one full scan may take, shared by its statements.""" + + def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None: + self.seconds = float(seconds) + self._clock = clock + self._start = clock() + + def elapsed(self) -> float: + return self._clock() - self._start + + def left(self) -> float: + return self.seconds - self.elapsed() + + def require(self, subject: str) -> float: + """The seconds left, or Refused naming `subject` once none are.""" + left = self.left() + if left <= 0: + raise Refused([_out_of_budget(subject, self.seconds)]) + return left + + +def _out_of_budget(subject: str, seconds: float) -> Refusal: + return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it") + + +def _on_cluster(topology: Topology) -> str: + return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else "" + + +async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None: + """A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries + `ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must + answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a + profile's `never_throw` hides none). One that does not finish in time refuses as the + budget when the budget is spent, else with the server's error and the hosts it did not + finish on: a replica that cannot sync.""" + seconds = max(1, int(budget.require(subject))) + # `receive_timeout` bounds SYNC REPLICA's own wait. + bounds = {"max_execution_time": seconds, "receive_timeout": seconds} + what = sql.split(" ON CLUSTER ")[0].split(" `")[0] + try: + if not topology.multi: + await ch.execute(sql, settings=bounds) + return + body = await ch.raw_bytes( + sql, + settings={ + **bounds, + "distributed_ddl_output_mode": "throw", + "distributed_ddl_task_timeout": seconds, + "default_format": "JSONEachRow", + }, + ) + rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status") + except Exception as e: # only a timeout is judged here; the rest pass to the caller + if error_code(e) not in _TIMEOUT_CODES: + raise + # The statement was given the budget's whole seconds, so up to one is left over. + if budget.left() <= 1: + raise Refused([_out_of_budget(subject, budget.seconds)]) from None + done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"} + unfinished = ", ".join(sorted(topology.hosts - done)) or "this server" + raise Refused( + [Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")] + ) from e + failed = [r for r in rows if int(r["status"]) != 0] + if failed: + raise Refused( + [Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed] + ) + answered = {str(r.get("host")) for r in rows} + if answered != topology.hosts: + missing = ", ".join(sorted(topology.hosts - answered)) or "none" + raise Refused( + [ + Refusal( + subject, f"the hosts that answered are not the listed ones (missing: {missing})" + ) + ] + ) + + +class BodyError(DatabaseError): + """An error ClickHouse wrote into a response body after it began streaming rows: a + server error as the driver raises one, its text keeping the server's `Code: .`, so it + is judged transient or not, and by code, as any other. `rows` are the rows before it.""" + + def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None: + super().__init__(text) + self.rows = list(rows or []) + + +def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]: + """The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that + is not one (an `{"exception": …}` row, or the text tail a server appends once a + statement fails after its first block), with the rest of the body as its text.""" + lines = list(lines) + rows: list[dict[str, Any]] = [] + for i, line in enumerate(lines): + if not line.strip(): + continue + try: + row = json.loads(line) + except ValueError: + row = None + if not isinstance(row, dict) or key not in row: + exception = row.get("exception") if isinstance(row, dict) else None + raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows) + rows.append(row) + return rows diff --git a/server/insights/store/cell_catalog.py b/server/insights/store/cell_catalog.py new file mode 100644 index 0000000000..e973826ae4 --- /dev/null +++ b/server/insights/store/cell_catalog.py @@ -0,0 +1,500 @@ +"""Complete catalog visibility, replica identity and grant checks for cell reads.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from server.core import ch_names +from server.core.cell_ch import ( + CLUSTER_SQL, + TAG, + WHOLE_SQL, + Refusal, + Refused, + Topology, + databases, + per_host, +) +from server.core.ch import query_tag +from server.core.ch_errors import error_code, error_line +from server.insights.domain.analysis import ANALYST_USER, OTHERS_POLICY_NAME, POLICY_NAME + +_ACCESS_DENIED = 497 # ClickHouse's ACCESS_DENIED + +_IMPLIED_BY: dict[str, tuple[str, ...]] = { + "SELECT": ("ALL",), + "SHOW USERS": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROLES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW ROW POLICIES": ("SHOW ACCESS", "ACCESS MANAGEMENT", "ALL"), + "SHOW TABLES": ("SHOW", "ALL"), + "SHOW COLUMNS": ("SHOW", "ALL"), + "REMOTE": ("SOURCES", "ALL"), + "CLUSTER": ("ALL",), + "SYSTEM SYNC REPLICA": ("SYSTEM", "ALL"), + "SYSTEM FLUSH ASYNC INSERT QUEUE": ("SYSTEM FLUSH", "SYSTEM", "ALL"), +} + + +@dataclass(frozen=True) +class Grant: + """One privilege a user holds: `database` None for a global grant, `table` None for the + whole database; a `database` ending in `*` is a prefix grant. `revoked` marks a partial + revoke, which narrows a grant: no grant counts it, and attach's own user refuses one.""" + + access: str + database: str | None = None + table: str | None = None + grant_option: bool = False + revoked: bool = False + + def covers(self, database: str | None) -> bool: + """Whether this grant reaches all of `database` (None: every database).""" + if self.table is not None: + return False + return self.database is None or (database is not None and _db_match(self, database)) + + def touches(self, database: str) -> bool: + """Whether this grant reaches any table of `database`.""" + return self.database is None or _db_match(self, database) + + +def _db_match(grant: Grant, database: str) -> bool: + assert grant.database is not None + if grant.database.endswith("*"): + return database.startswith(grant.database[:-1]) + return grant.database == database + + +def holds(grants: Iterable[Grant], access: str, database: str | None = None) -> bool: + """Whether `grants` give `access` on all of `database` (None: globally).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return any(g.access in wanted and g.covers(database) and not g.revoked for g in grants) + + +def narrowing(grants: Iterable[Grant], access: str, database: str | None = None) -> list[Grant]: + """The partial revokes in `grants` of `access`, or of a grant implying it, on any part + of `database` (None: anywhere).""" + wanted = {access, *_IMPLIED_BY.get(access, ())} + return [ + g + for g in grants + if g.revoked + and g.access in wanted + and (database is None or g.database is None or _db_match(g, database)) + ] + + +_GRANT = re.compile( + r"^(?PGRANT|REVOKE) (?P.+?) ON (?P(?:`(?:[^`\\]|\\.)*`|[^\s`])+) " + r"(?:TO|FROM) \S.*?(?P WITH GRANT OPTION)?$" +) + +_IDENT = re.compile(r"`((?:[^`\\]|\\.)*)`|([^.`]+)") + + +def _split_privileges(text: str) -> list[str]: + """`SELECT(a, b), INSERT` → [`SELECT`, `INSERT`]: commas inside a column list stay.""" + out, depth, start = [], 0, 0 + for i, ch in enumerate(text): + depth += {"(": 1, ")": -1}.get(ch, 0) + if ch == "," and depth == 0: + out.append(text[start:i]) + start = i + 1 + out.append(text[start:]) + return [p.split("(")[0].strip() for p in out if p.strip()] + + +def _target(on: str) -> tuple[str | None, str | None] | str: + """`*.*` and `*` → (None, None), `db.*` → (db, None), `db.t` → (db, t); a source a + `READ` grant names (`REMOTE`) as itself.""" + if on == "*": + return (None, None) + parts = [a or b for a, b in _IDENT.findall(on)] + if len(parts) != 2: + return on + db, table = parts + return (None if db == "*" else db, None if table == "*" else table) + + +def parse_show_grants(lines: Iterable[str]) -> list[Grant]: + """The grants of `SHOW GRANTS FINAL` (attach's own), whose lines already hold its + roles', and its partial revokes (`revoked`). Role grants (`GRANT r TO u`) and the + revokes of a grant option alone are left out.""" + out = [] + for line in lines: + m = _GRANT.match(line.strip()) + if m is None or m["privs"].startswith("GRANT OPTION FOR "): + continue + target = _target(m["on"]) + option, revoked = bool(m["opt"]), m["verb"] == "REVOKE" + for access in _split_privileges(m["privs"]): + if isinstance(target, str): + # `READ ON REMOTE` is printed `REMOTE ON *.*`; a newer server may print it + # as written. + if access in ("READ", "WRITE"): + out.append(Grant(target, grant_option=option, revoked=revoked)) + continue + out.append(Grant(access, target[0], target[1], option, revoked)) + return out + + +def resolve_grants( + user: str, + grant_rows: Iterable[Mapping[str, Any]], + role_rows: Iterable[Mapping[str, Any]], + *, + default_only: bool = False, +) -> list[Grant]: + """`user`'s grants from one host's `system.grants` and `system.role_grants` rows, every + role it holds followed (default or not, since it can set any). `default_only` follows + only the user's default roles, the ones a new session has (attach's own reads run in + one), and every role those hold: `granted_role_is_default` marks the user's own grants, + while a role's grant of another always passes it on.""" + # A user and a role may share a name: each holder is keyed by its kind. + granted: dict[tuple[str, str], set[str]] = {} + for r in role_rows: + holder = ( + ("user", str(r["user_name"])) + if r.get("user_name") + else ("role", str(r.get("role_name"))) + ) + if default_only and holder[0] == "user" and r.get("granted_role_is_default") == 0: + continue + granted.setdefault(holder, set()).add(str(r["granted_role_name"])) + roles: set[str] = set() + todo = list(granted.get(("user", user), ())) + while todo: + role = todo.pop() + if role not in roles: + roles.add(role) + todo.extend(granted.get(("role", role), ())) + out = [] + for r in grant_rows: + if r.get("user_name") == user or (r.get("role_name") in roles and not r.get("user_name")): + out.append(_row_grant(r)) + return out + + +def _row_grant(r: Mapping[str, Any]) -> Grant: + """A `system.grants` row as a Grant: `READ ON REMOTE` is access `READ` on object + `REMOTE` there, read as `REMOTE` (as `SHOW GRANTS` prints it), and a wildcard database + as a prefix.""" + access = str(r["access_type"]) + if access in ("READ", "WRITE") and r.get("access_object"): + access = str(r["access_object"]) + database = None if r.get("database") is None else str(r["database"]) + if database is not None and r.get("is_wildcard") and not database.endswith("*"): + database += "*" + return Grant( + access, + database, + None if r.get("table") is None else str(r["table"]), + bool(r.get("grant_option")), + bool(r.get("is_partial_revoke")), + ) + + +def own_problems(grants: list[Grant], topology: Topology, dbs: list[str]) -> list[str]: + """What attach's own user lacks to read what the check and the scan read.""" + needs: list[tuple[str, str | None]] = [ + ("SELECT", "system"), + ("SHOW USERS", None), + ("SHOW ROLES", None), + ("SHOW ROW POLICIES", None), + ("SHOW TABLES", None), + ("SHOW COLUMNS", None), + ("SYSTEM FLUSH ASYNC INSERT QUEUE", None), + ] + for db in dbs: + needs.append(("SELECT", db)) + if topology.hosts: + needs.append(("SYSTEM SYNC REPLICA", db)) + if topology.multi: + needs += [("REMOTE", None), ("CLUSTER", None)] + out = [] + for access, scope in needs: + if not holds(grants, access, scope): + out.append(_label(access, scope)) + elif revokes := narrowing(grants, access, scope): + # A revoke hides part of what the checks read, which reads short silently. + out.append( + f"{_label(access, scope)} (narrowed by REVOKE " + + ", ".join(f"{g.access} ON {_scope(g)}" for g in revokes) + + ")" + ) + return out + + +def _label(access: str, db: str | None) -> str: + if access == "REMOTE": + return "READ ON REMOTE" + return f"{access} ON {db}.*" if db else f"{access} ON *.*" + + +def _scope(g: Grant) -> str: + if g.database is None: + return "*.*" + return f"{g.database}.{g.table or '*'}" + + +def extra_replicas(rows: Iterable[Mapping[str, Any]]) -> dict[str, set[str]]: + """Per replicated table (`db.table`), the replicas Keeper lists that no host of the + cluster is (`replica_is_active` keys against every host's `replica_name`, by + `zookeeper_path`).""" + ours: dict[str, set[str]] = {} + listed: dict[str, set[str]] = {} + names: dict[str, str] = {} + for r in rows: + path = str(r["zookeeper_path"]) + ours.setdefault(path, set()).add(str(r["replica_name"])) + listed.setdefault(path, set()).update(str(k) for k in (r["replica_is_active"] or {})) + names.setdefault(path, f"{r['database']}.{r['table']}") + out = {} + for path, keys in listed.items(): + if extra := keys - ours[path]: + out[names[path]] = extra + return out + + +async def _own_user(ch: Any) -> str: + row = await ch.query_one(f"SELECT currentUser() AS u {WHOLE_SQL}") + return str((row or {}).get("u") or "") + + +async def _show_grants(ch: Any) -> list[Grant]: + """Attach's own grants, as `SHOW GRANTS FINAL` prints them: the one read that needs no + privilege, so a user lacking `SELECT ON system.*` is told what it lacks.""" + # Pinned: a profile's `break` could end the list before its REVOKE lines. + rows = await ch.query_all(f"SHOW GRANTS FINAL {WHOLE_SQL}") + return parse_show_grants(str(next(iter(r.values()))) for r in rows if r) + + +HostGrants = dict[str, dict[str, list[Grant]]] + + +async def host_names(ch: Any, topology: Topology) -> dict[str, str]: + """Each answering host's OS hostname (an opaque group key) and the one configured name + it answers for in the cluster; Refused unless there is one group per listed host, each + with exactly one name, the names together the listing. The topology's own refusals (two + servers under one host name, several shards) come first, then what attach's own user + lacks by its own `SHOW GRANTS FINAL` (this read needs READ ON REMOTE), before any read + through `clusterAllReplicas`.""" + if refusals := topology.refusals(): + raise Refused(refusals) + with query_tag(TAG): + if lacking := await first_refusals(ch, topology): + raise Refused(lacking) + rows = await ch.query_all( + "SELECT hostName() AS h, groupArrayIf(host_name, is_local) AS names " + "FROM clusterAllReplicas({c:String}, system.clusters) WHERE cluster = {c:String} " + f"GROUP BY h {CLUSTER_SQL}", + {"c": topology.cluster}, + ) + where = f"cluster {topology.cluster}" + refusals = [] + out: dict[str, str] = {} + for r in rows: + names = sorted(str(n) for n in r["names"]) + if not names: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host's address in {where} does not resolve to a local interface", + ) + ) + elif len(names) > 1: + refusals.append( + Refusal( + f"host {r['h']}", + f"this host answers for several entries in {where} ({', '.join(names)}): " + "two local entries, or two hosts with one OS hostname", + ) + ) + else: + out[str(r["h"])] = names[0] + named = sorted(out.values()) + if not refusals and (len(named) != len(set(named)) or set(named) != topology.hosts): + missing = sorted(topology.hosts - set(named)) + refusals.append( + Refusal( + where, + "the answering hosts are not the listed ones" + + (f" (no answer for {', '.join(missing)})" if missing else ""), + ) + ) + if refusals: + raise Refused(refusals) + return out + + +async def first_refusals(ch: Any, topology: Topology) -> list[Refusal]: + """What attach's own user lacks by its own `SHOW GRANTS FINAL`, the one read that needs + no privilege. It prints the grants of every role the user holds, a role a session does + not have included: this first look names what is missing; the rows decide.""" + if missing := own_problems(await _show_grants(ch), topology, databases()): + own = await _own_user(ch) + return [Refusal(f"user {own}", f"attach's own user lacks {', '.join(missing)}")] + return [] + + +async def own_refusals( + ch: Any, topology: Topology, names: Mapping[str, str] +) -> tuple[list[Refusal], HostGrants]: + """What attach's own user lacks to read what the checks read (without it `system.tables` + and `system.columns` would silently list less than there is), and every user's grants + per host once they could be read. First its own `SHOW GRANTS FINAL`; then, on several + hosts, that every host runs the reads as this user; then that user's grants as each host + keeps them; then the row policies, which would hide rows from its scan.""" + dbs = databases() + if lacking := await first_refusals(ch, topology): + return lacking, {} + own = await _own_user(ch) + try: + if topology.multi and (wrong := await _process_users(ch, topology, names, own)): + return wrong, {} + per_user, own_grants = await _host_grants(ch, topology, names, own) + out = [] + for host, grants in sorted(own_grants.items()): + # Each host keeps its own users: attach's must hold the same on every one, + # through its default roles alone, as its sessions have them. + if lacks := own_problems(grants, topology, dbs): + out.append(Refusal(f"user {own}", f"lacks {', '.join(lacks)}{_on(host)}")) + if not out: + out = await _row_policies(ch, topology, names, own, dbs) + except Exception as e: # only a refused read is judged here; the rest pass to the caller + if error_code(e) != _ACCESS_DENIED: + raise + reason = ( + f"attach's own user cannot read what the checks read ({error_line(e)}); a grant " + "held only through a role that is not a default role does not reach its sessions" + ) + return [Refusal(f"user {own}", reason)], {} + return out, per_user + + +async def _row_policies( + ch: Any, topology: Topology, names: Mapping[str, str], own: str, dbs: list[str] +) -> list[Refusal]: + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, short_name, database, table, select_filter, is_restrictive, " + "apply_to_all, apply_to_list, apply_to_except FROM {source:row_policies} " + "WHERE database IN {dbs:Array(String)} OR database = 'system'", + {"dbs": dbs}, + ) + out = [] + for host, found in sorted(rows.items()): + for subject, why in row_policy_problems(found, own): + out.append(Refusal(subject, f"{why}{_on(host)}")) + return out + + +def _as_loom_writes(r: Mapping[str, Any]) -> bool: + """Whether one of Loom's two policies names exactly whom Loom writes it for: a role in + either list could cover or exclude attach's user without naming it.""" + if r["short_name"] == POLICY_NAME: + return not r.get("apply_to_all") and list(r.get("apply_to_list") or []) == [ANALYST_USER] + return bool(r.get("apply_to_all")) and list(r.get("apply_to_except") or []) == [ANALYST_USER] + + +def _applies(r: Mapping[str, Any], user: str) -> bool: + if r.get("apply_to_all"): + return user not in (r.get("apply_to_except") or []) + return user in (r.get("apply_to_list") or []) + + +def row_policy_problems(rows: Iterable[Mapping[str, Any]], own: str) -> list[tuple[str, str]]: + """What hides rows from attach's own user, per row policy on a table of the configured + databases or a system table. Any policy on a system table refuses: the checks read them. + Once a table has a row policy, a user that no permissive policy covers reads none of its + rows, and a restrictive one filters them: either shortens the scan silently. Loom's own + pair passes (`analysis.POLICY_NAME` for the analyst, and `analysis.OTHERS_POLICY_NAME` + letting every other user read every row) while it covers attach's user as written; every + other policy refuses.""" + tables: dict[str, list[Mapping[str, Any]]] = {} + for r in rows: + tables.setdefault(f"{ch_names.logical(str(r['database']))}.{r['table']}", []).append(r) + out = [] + for table, policies in sorted(tables.items()): + subject = f"row policy on {table}" + if table.startswith("system."): + # The checks read the system tables: a policy there hides what they read. Loom + # writes none. + out += [ + (f"{subject}: {p['short_name']}", "a row policy hides rows from the checks") + for p in policies + ] + continue + others = [p for p in policies if p["short_name"] == OTHERS_POLICY_NAME] + for p in policies: + name = str(p["short_name"]) + if name not in (POLICY_NAME, OTHERS_POLICY_NAME): + out.append((f"{subject}: {name}", "a row policy hides rows from the scan")) + elif not _as_loom_writes(p): + out.append((f"{subject}: {name}", "names other users or roles than Loom writes")) + elif name == POLICY_NAME and _applies(p, own): + out.append((f"{subject}: {name}", f"the analyst's policy applies to {own}")) + if not any( + not p.get("is_restrictive") and str(p.get("select_filter")) == "1" and _applies(p, own) + for p in others + ): + out.append((subject, f"no permissive policy `USING 1` covers {own}: it reads no row")) + return out + + +async def _process_users( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> list[Refusal]: + """On several hosts: each host runs this read as attach's own user, exactly once.""" + rows = await per_host( + ch, + topology, + names, + "SELECT hostName() AS h, user FROM {source:processes} " + "WHERE initial_query_id = initialQueryID()", + ) + out = [] + for host, found in sorted(rows.items()): + users = [str(r["user"]) for r in found] + if users != [own]: + out.append( + Refusal( + f"host {host}", + f"runs attach's reads as {', '.join(users) or 'no user'}, not {own}: set an " + "interserver secret, or the cluster's user to attach's", + ) + ) + return out + + +async def _host_grants( + ch: Any, topology: Topology, names: Mapping[str, str], own: str +) -> tuple[HostGrants, dict[str, list[Grant]]]: + """Per host, each user it defines and that user's grants, from `system.grants` and + `system.role_grants` on every topology (every granted role followed, default or not); + and per host the grants of `own` through its default roles alone.""" + users = await per_host(ch, topology, names, "SELECT hostName() AS h, name FROM {source:users}") + # Every column: `access_object` and `is_wildcard` are read where the server has them. + grants = await per_host(ch, topology, names, "SELECT hostName() AS h, * FROM {source:grants}") + roles = await per_host( + ch, topology, names, "SELECT hostName() AS h, * FROM {source:role_grants}" + ) + every = { + host: { + str(u["name"]): resolve_grants(str(u["name"]), grants[host], roles[host]) for u in found + } + for host, found in users.items() + } + own_grants = { + host: resolve_grants(own, grants[host], roles[host], default_only=True) for host in users + } + return every, own_grants + + +def _on(host: str) -> str: + return f" on host {host}" if host else "" diff --git a/tests/integration/test_cell_attach.py b/tests/integration/test_cell_attach.py index 4531ef81a7..06b8002c38 100644 --- a/tests/integration/test_cell_attach.py +++ b/tests/integration/test_cell_attach.py @@ -29,9 +29,9 @@ from server.cli import cell as cell_cli from server.cli import cell_attach, cell_attach_pg, cell_prepare -from server.cli.cell_ch import Refused -from server.core import cell, cell_probe, cell_triggers, instances, pg +from server.core import cell, cell_probe, cell_sync, cell_triggers, instances, pg from server.core import ch as ch_core +from server.core.cell_ch import Refused from server.core.schema import pg_migrate from server.router import app as router_app from server.router import directory @@ -895,9 +895,9 @@ async def clickhouse_inventory(*_args: Any) -> tuple[list[Any], list[Any]]: monkeypatch.setattr(cell_scan, "list_clickhouse", clickhouse_inventory) clock = [0.0] - real_budget = cell_scan.Budget + real_budget = cell_sync.Budget monkeypatch.setattr( - cell_scan, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) + cell_sync, "Budget", lambda seconds: real_budget(seconds, clock=lambda: clock[0]) ) # The budget runs out on the largest table, as it does on a real cell when that table's # read is what is too slow: every smaller one was read first. diff --git a/tests/integration/test_cell_scan.py b/tests/integration/test_cell_scan.py index c78bc40331..32a9560c6f 100644 --- a/tests/integration/test_cell_scan.py +++ b/tests/integration/test_cell_scan.py @@ -17,11 +17,13 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused -from server.cli.cell_scan import Budget, Table, TableList -from server.core import cell, ch_names, pg +from server.cli import cell_scan, cell_writers +from server.cli.cell_scan import Table, TableList +from server.core import cell, cell_ch, ch_names, pg +from server.core.cell_ch import Refused +from server.core.cell_sync import Budget from server.core.ch import get_client +from server.insights.store import cell_catalog from server.settings import settings from tests.integration._pg_database import own_database from tests.integration._resources import unique_org @@ -287,12 +289,12 @@ async def test_grants_are_read_as_the_server_keeps_them( await ch.execute(f"GRANT {role} TO {users['writer']}") await ch.execute(f"SET DEFAULT ROLE NONE TO {users['writer']}") topology = await cell_ch.read_topology(ch) - every, _ = await cell_writers._host_grants(ch, topology, {}, "") + every, _ = await cell_catalog._host_grants(ch, topology, {}, "") (grants,) = every.values() - assert f"SELECT ON {schema}.*" in cell_writers.own_problems( + assert f"SELECT ON {schema}.*" in cell_catalog.own_problems( grants[users["narrow"]], topology, [schema] ) - assert cell_writers.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] + assert cell_catalog.holds(grants[users["remote"]], "REMOTE"), grants[users["remote"]] assert cell_writers.writer_reason(grants[users["writer"]], [schema]) == ( f"holds INSERT on {schema}.*" ) @@ -516,7 +518,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai created.remove(theirs) # Loom's own pair: the analyst's scope, and every other user reading every row, with # a scratch user standing in for the analyst. - monkeypatch.setattr(cell_writers, "ANALYST_USER", other) + monkeypatch.setattr(cell_catalog, "ANALYST_USER", other) for name, using, to in ( (POLICY_NAME, "org = ''", other), (OTHERS_POLICY_NAME, "1", f"ALL EXCEPT {other}"), @@ -527,7 +529,7 @@ async def test_a_row_policy_on_a_configured_table_refuses_unless_it_is_looms_pai ) created.append(name) counted = await ch.query_one(f"SELECT count() AS n FROM {schema}.memory") - refusals = await cell_writers._row_policies(ch, topology, {}, own, cell_ch.databases()) + refusals = await cell_catalog._row_policies(ch, topology, {}, own, cell_ch.databases()) finally: for name in created: await ch.execute(f"DROP ROW POLICY IF EXISTS `{name}` ON {schema}.memory") diff --git a/tests/integration/test_cell_unset_gate_pg.py b/tests/integration/test_cell_unset_gate_pg.py index 37e80a7057..e2ab59c65b 100644 --- a/tests/integration/test_cell_unset_gate_pg.py +++ b/tests/integration/test_cell_unset_gate_pg.py @@ -251,7 +251,7 @@ async def test_loom_cell_removes_check_reads_the_first_cells_postgres_alone( import httpx from server.cli import cell as cell_cli - from server.cli.cell_ch import Refused + from server.core.cell_ch import Refused async with own_database() as dsn: monkeypatch.setattr(settings, "pg_dsn", dsn) diff --git a/tests/unit/test_cell_attach.py b/tests/unit/test_cell_attach.py index a289fb9b7f..6862f8ea0d 100644 --- a/tests/unit/test_cell_attach.py +++ b/tests/unit/test_cell_attach.py @@ -18,9 +18,9 @@ from server.cli import cell_attach, cell_control, cell_scan from server.cli import cell_attach_beats as beats from server.cli import cell_attach_pg as apg -from server.cli.cell_ch import Refusal, Refused from server.cli.cell_scan import Table from server.core import cell, instances +from server.core.cell_ch import Refusal, Refused from server.router.limits import ATTACH_BATCH, MAX_BODY_BYTES from server.settings import settings from tests.unit.cell_control_owner import owner as _owner @@ -1443,8 +1443,8 @@ async def test_the_stray_read_names_a_refused_table_and_reads_on( ) -> None: from contextlib import asynccontextmanager - from server.cli import cell_ch, doctor_cell - from server.core import pg + from server.cli import doctor_cell + from server.core import cell_ch, pg async def topology(_ch: Any) -> Any: return cell_ch.Topology("", 0) diff --git a/tests/unit/test_cell_attach_serving_identity.py b/tests/unit/test_cell_attach_serving_identity.py index 469ed16ea3..f63f195fda 100644 --- a/tests/unit/test_cell_attach_serving_identity.py +++ b/tests/unit/test_cell_attach_serving_identity.py @@ -10,8 +10,8 @@ import pytest from server.cli import cell_attach, cell_control -from server.cli.cell_ch import Refused from server.core import pg +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FakeConn, _args diff --git a/tests/unit/test_cell_scan.py b/tests/unit/test_cell_scan.py index 56bf01a172..0366a44455 100644 --- a/tests/unit/test_cell_scan.py +++ b/tests/unit/test_cell_scan.py @@ -15,10 +15,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Refused, Topology -from server.cli.cell_scan import Budget, ScanResult, Table, TableList -from server.core import cell, pg +from server.cli import cell_scan +from server.cli.cell_scan import ScanResult, Table, TableList +from server.core import cell, cell_ch, pg +from server.core.cell_ch import Refused, Topology +from server.core.cell_sync import Budget +from server.insights.store import cell_catalog ONE = Topology("default", 1, (("localhost", 9000),)) TWO = Topology("main", 1, (("ch-0", 9000), ("ch-1", 9000))) @@ -178,7 +180,7 @@ async def test_each_host_answers_for_exactly_one_listed_name() -> None: [{"h": "os-0", "names": ["ch-0"]}, {"h": "os-1", "names": ["ch-1"]}], ) ) - assert await cell_writers.host_names(ch, TWO) == NAMES + assert await cell_catalog.host_names(ch, TWO) == NAMES # Every clusterAllReplicas read waits for every replica and reads its own host locally. reads = [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] assert "skip_unavailable_shards = 0, prefer_localhost_replica = 1" in reads[0] @@ -190,7 +192,7 @@ async def test_a_user_without_read_on_remote_is_named_before_any_cluster_read() ] ch = _granted(FakeCH(), narrow) with pytest.raises(Refused, match="attach's own user lacks READ ON REMOTE"): - await cell_writers.host_names(ch, TWO) + await cell_catalog.host_names(ch, TWO) assert not [s for s in ch.sql("query_all") if "clusterAllReplicas" in s] @@ -208,7 +210,7 @@ async def test_a_host_without_one_configured_name_refuses( rows: list[dict[str, Any]], reason: str ) -> None: with pytest.raises(Refused, match=reason): - await cell_writers.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) + await cell_catalog.host_names(_granted(FakeCH().on("clusterAllReplicas", rows)), TWO) # -- the table list ---------------------------------------------------------------------- @@ -737,7 +739,7 @@ async def test_two_servers_under_one_host_name_refuse_before_any_read() -> None: same = Topology("main", 1, (("ch-0", 9000), ("ch-0", 9001))) ch = FakeCH() with pytest.raises(Refused, match="lists host ch-0 2 times"): - await cell_writers.host_names(ch, same) + await cell_catalog.host_names(ch, same) listed = await cell_scan.list_tables(ch, None, same, {}) assert [r.reason for r in listed.refusals] == [ "lists host ch-0 2 times (ports 9000, 9001): give each server its own host name" diff --git a/tests/unit/test_cell_scim_precondition.py b/tests/unit/test_cell_scim_precondition.py index 2c0bb0946f..0cf91e4792 100644 --- a/tests/unit/test_cell_scim_precondition.py +++ b/tests/unit/test_cell_scim_precondition.py @@ -11,8 +11,8 @@ from server.auth import authz, revocation, scim, subject_revocation from server.cli import cell_attach, cell_attach_beats -from server.cli.cell_ch import Refused from server.core import cell, ch +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_attach import FIELDS, FakeConn, _args, _beat, _value from tests.unit.test_scim_people import People diff --git a/tests/unit/test_cell_scim_precondition_failures.py b/tests/unit/test_cell_scim_precondition_failures.py index 11f46f0f60..ed175cc289 100644 --- a/tests/unit/test_cell_scim_precondition_failures.py +++ b/tests/unit/test_cell_scim_precondition_failures.py @@ -9,7 +9,7 @@ from server.auth import revocation from server.auth.revocation import scim_revoke as actual_scim_revoke -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from tests.unit.test_cell_attach_serving_identity import serving_run as serving_run from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_update_participants.py b/tests/unit/test_cell_update_participants.py index 5465767496..353dc1e2cf 100644 --- a/tests/unit/test_cell_update_participants.py +++ b/tests/unit/test_cell_update_participants.py @@ -196,15 +196,15 @@ async def test_census_syncs_before_complete_replica_union(monkeypatch, engine): from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology events = [] topology = Topology("replicas", 1, (("a", 9000), ("b", 9000))) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(ch, top, names, sql, params): if "source:tables" in sql: @@ -235,7 +235,7 @@ async def query(sql, params): return [participant(1), participant(2), participant(1, "drained", version=2)] monkeypatch.setattr(module, "per_host", per_host) - monkeypatch.setattr(module.cell_scan, "_system", sync) + monkeypatch.setattr(module.cell_sync, "_system", sync) result = await module.census(SimpleNamespace(query_all=query)) assert len(result) == 2 and result[str(UUID(int=1))]["state"] == "drained" assert events == (["sync", "read"] if engine.startswith("Replicated") else ["read"]) @@ -251,7 +251,7 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey from unittest.mock import AsyncMock from server.cli import cell_update_participants as module - from server.cli.cell_ch import Topology + from server.core.cell_ch import Topology topology = Topology( "replicas", @@ -260,9 +260,9 @@ async def test_incomplete_topology_or_sync_never_returns_a_partial_census(monkey ) monkeypatch.setattr(module, "read_topology", AsyncMock(return_value=topology)) monkeypatch.setattr( - module.cell_writers, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) + module.cell_catalog, "host_names", AsyncMock(return_value={"a": "a", "b": "b"}) ) - monkeypatch.setattr(module.cell_writers, "own_refusals", AsyncMock(return_value=([], []))) + monkeypatch.setattr(module.cell_catalog, "own_refusals", AsyncMock(return_value=([], []))) async def per_host(*args): if problem == "unreachable": @@ -293,7 +293,7 @@ async def per_host(*args): monkeypatch.setattr(module, "per_host", per_host) monkeypatch.setattr( - module.cell_scan, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) + module.cell_sync, "_system", AsyncMock(side_effect=OSError("cannot synchronize")) ) query = AsyncMock(side_effect=AssertionError("must not read partial census")) with pytest.raises((module.cell_control.Refused, OSError)): diff --git a/tests/unit/test_cell_update_scim_runtime.py b/tests/unit/test_cell_update_scim_runtime.py index 2cbb67094d..0fed38129c 100644 --- a/tests/unit/test_cell_update_scim_runtime.py +++ b/tests/unit/test_cell_update_scim_runtime.py @@ -12,9 +12,9 @@ from server.auth import scim from server.cli import cell_control, cell_update_participants, cell_update_probe, cell_update_run -from server.cli.cell_ch import Refused from server.cli.cell_update_pg import State from server.core import cell, pg_profile +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.cell_control_owner import owner from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run diff --git a/tests/unit/test_cell_writers.py b/tests/unit/test_cell_writers.py index a3e85bca42..e1c84a457c 100644 --- a/tests/unit/test_cell_writers.py +++ b/tests/unit/test_cell_writers.py @@ -11,9 +11,12 @@ import pytest -from server.cli import cell_ch, cell_scan, cell_writers -from server.cli.cell_ch import Topology -from server.cli.cell_writers import Allowed, Grant, holds, parse_show_grants, writer_reason +from server.cli import cell_scan, cell_writers +from server.cli.cell_writers import Allowed, writer_reason +from server.core import cell_ch +from server.core.cell_ch import Topology +from server.insights.store import cell_catalog +from server.insights.store.cell_catalog import Grant, holds, parse_show_grants from tests.unit.test_cell_scan import NAMES, ONE, TWO, FakeCH DBS = ["loom", "loom_telemetry"] @@ -69,7 +72,7 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: "REVOKE SELECT ON loom.secret FROM attach", "REVOKE GRANT OPTION FOR SELECT ON other.* FROM attach", ] - assert cell_writers.own_problems(parse_show_grants(lines), ONE, DBS) == [ + assert cell_catalog.own_problems(parse_show_grants(lines), ONE, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW TABLES ON side.*)", "SELECT ON loom.* (narrowed by REVOKE SELECT ON loom.secret)", ] @@ -90,8 +93,8 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: rows.append( {"user_name": "attach", "access_type": "SHOW", "database": "side", "is_partial_revoke": 1} ) - grants = cell_writers.resolve_grants("attach", rows, []) - assert cell_writers.own_problems(grants, TWO, DBS) == [ + grants = cell_catalog.resolve_grants("attach", rows, []) + assert cell_catalog.own_problems(grants, TWO, DBS) == [ "SHOW TABLES ON *.* (narrowed by REVOKE SHOW ON side.*)", "SHOW COLUMNS ON *.* (narrowed by REVOKE SHOW ON side.*)", ] @@ -101,10 +104,10 @@ def test_a_partial_revoke_narrowing_attachs_own_reads_refuses() -> None: def test_attachs_own_user_needs_every_read_its_checks_make() -> None: full = [Grant("SELECT"), Grant("SHOW"), Grant("SHOW ACCESS"), Grant("SYSTEM")] - assert cell_writers.own_problems(full, ONE, DBS) == [] - assert cell_writers.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] + assert cell_catalog.own_problems(full, ONE, DBS) == [] + assert cell_catalog.own_problems(full, TWO, DBS) == ["READ ON REMOTE", "CLUSTER ON *.*"] narrow = [Grant("SELECT", "loom", "memory"), Grant("SELECT", "system"), Grant("SHOW")] - assert cell_writers.own_problems(narrow, Topology("", 0), DBS) == [ + assert cell_catalog.own_problems(narrow, Topology("", 0), DBS) == [ "SHOW USERS ON *.*", "SHOW ROLES ON *.*", "SHOW ROW POLICIES ON *.*", @@ -168,7 +171,7 @@ def test_a_grants_row_as_newer_servers_write_it() -> None: {"user_name": "u", "access_type": "WRITE", "access_object": "URL", "database": None}, {"user_name": "u", "access_type": "INSERT", "database": "lo", "is_wildcard": 1}, ] - assert cell_writers.resolve_grants("u", rows, []) == [ + assert cell_catalog.resolve_grants("u", rows, []) == [ Grant("REMOTE"), Grant("URL"), Grant("INSERT", "lo*"), @@ -190,9 +193,9 @@ def test_attachs_own_sessions_have_only_its_default_roles() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "spare", "granted_role_is_default": 0}, ] # fmt: skip - every = cell_writers.resolve_grants("u", grant_rows, role_rows) + every = cell_catalog.resolve_grants("u", grant_rows, role_rows) assert {g.access for g in every} == {"SHOW", "SELECT", "INSERT"} - session = cell_writers.resolve_grants("u", grant_rows, role_rows, default_only=True) + session = cell_catalog.resolve_grants("u", grant_rows, role_rows, default_only=True) assert {g.access for g in session} == {"SHOW", "SELECT"} @@ -203,7 +206,7 @@ def test_a_role_named_as_a_user_is_not_that_user() -> None: ] # Role `attach` holds role `w`; user `attach` holds neither. role_rows = [{"user_name": None, "role_name": "attach", "granted_role_name": "w"}] - assert cell_writers.resolve_grants("attach", grant_rows, role_rows) == [] + assert cell_catalog.resolve_grants("attach", grant_rows, role_rows) == [] def test_roles_are_followed_and_partial_revokes_ignored() -> None: @@ -224,7 +227,7 @@ def test_roles_are_followed_and_partial_revokes_ignored() -> None: {"user_name": "u", "role_name": None, "granted_role_name": "outer"}, {"user_name": None, "role_name": "outer", "granted_role_name": "inner"}, ] - assert cell_writers.resolve_grants("u", grant_rows, role_rows) == [ + assert cell_catalog.resolve_grants("u", grant_rows, role_rows) == [ Grant("INSERT", "loom"), Grant("SELECT"), Grant("SELECT", "loom", "secret", revoked=True), @@ -470,7 +473,7 @@ def test_a_replica_keeper_lists_outside_the_cluster_is_extra() -> None: "replica_is_active": {"r0": 1}, }, ] - assert cell_writers.extra_replicas(rows) == {"loom.memory": {"dr"}} + assert cell_catalog.extra_replicas(rows) == {"loom.memory": {"dr"}} def _row(user: str, access: str, database: str | None = None) -> dict[str, Any]: @@ -775,7 +778,7 @@ def test_loom_s_own_row_policy_pair_lets_attach_read_every_row() -> None: _policy(POLICY_NAME, select_filter="org = 'a'", apply_to_list=["loom_analyst"]), _policy(OTHERS_POLICY_NAME, apply_to_all=1, apply_to_except=["loom_analyst"]), ] - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] def test_any_row_policy_on_a_system_table_refuses() -> None: @@ -783,7 +786,7 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: _policy("own", database="system", table="grants", apply_to_all=1), _policy("loom_analyst_others", database="system", table="tables", apply_to_all=1), ] - assert cell_writers.row_policy_problems(rows, "attach") == [ + assert cell_catalog.row_policy_problems(rows, "attach") == [ ("row policy on system.grants: own", "a row policy hides rows from the checks"), ( "row policy on system.tables: loom_analyst_others", @@ -839,13 +842,13 @@ def test_any_row_policy_on_a_system_table_refuses() -> None: def test_a_row_policy_hiding_rows_from_attach_refuses( rows: list[dict[str, Any]], reason: str ) -> None: - found = cell_writers.row_policy_problems(rows, "attach") + found = cell_catalog.row_policy_problems(rows, "attach") assert any(reason in why for _, why in found), found async def test_the_row_policies_read_covers_the_system_tables() -> None: ch = _cluster() - await cell_writers.own_refusals(ch, TWO, NAMES) + await cell_catalog.own_refusals(ch, TWO, NAMES) (sql,) = [s for s in ch.sql("query_all") if "system.row_policies" in s] assert "database IN {dbs:Array(String)} OR database = 'system'" in sql @@ -924,14 +927,14 @@ def _looms_policies() -> list[dict[str, Any]]: def test_the_policies_loom_s_analyst_setup_writes_pass() -> None: rows = _looms_policies() assert {r["table"] for r in rows} >= {"memory", "analysis_canary"} - assert cell_writers.row_policy_problems(rows, "attach") == [] + assert cell_catalog.row_policy_problems(rows, "attach") == [] # Without every other user's policy, the canary would read no row for attach. alone = [ r for r in rows if not (r["table"] == "analysis_canary" and r["short_name"] == "loom_analyst_others") ] - assert cell_writers.row_policy_problems(alone, "attach") == [ + assert cell_catalog.row_policy_problems(alone, "attach") == [ ( "row policy on loom.analysis_canary", "no permissive policy `USING 1` covers attach: it reads no row", diff --git a/tests/unit/test_scim_review118_regressions.py b/tests/unit/test_scim_review118_regressions.py index 1df99b7173..182c13eaba 100644 --- a/tests/unit/test_scim_review118_regressions.py +++ b/tests/unit/test_scim_review118_regressions.py @@ -7,7 +7,7 @@ import pytest from server.auth import aliases, jit, login, oauth -from server.cli.cell_ch import Refused +from server.core.cell_ch import Refused from server.settings import settings from tests.unit.test_cell_scim_precondition import provisioned_run as provisioned_run from tests.unit.test_scim_people import people as people