Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/architecture/cells.md
Original file line number Diff line number Diff line change
Expand Up @@ -1716,7 +1716,7 @@ from its cache ([upgrades](../operations/upgrades.md#upgrade-procedure)).
cluster listing the same, and the distinct orgs of each table outside `LOG_TABLES`;
`server/cli/cell_writers.py`: the ClickHouse writer check (the users, user directories,
objects and replicas outside the operator's allow lists);
`server/cli/cell_ch.py`: what both share (the cluster's hosts, the per-host reads);
`server/core/cell_ch.py`: what both share (the cluster's hosts, the per-host reads);
- `schemas/pg_migrations/008_cells.sql`, `schemas/pg_migrations/009_cell_probe.sql` and
`schemas/migrations/094_cell_heartbeats.sql`: the cell's tables;
- `server/auth/tenancy.py` (`ensure_held`): the tenancy check and the placement call,
Expand Down
9 changes: 5 additions & 4 deletions server/cli/cell_attach.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@
from server.cli import cell_attach_pg as apg
from server.cli import cell_control, cell_scan, cell_scim
from server.cli._ch import writes
from server.cli.cell_ch import Refusal, Refused, read_topology
from server.core import cell, cell_triggers
from server.core import cell, cell_sync, cell_triggers
from server.core.cell_ch import Refusal, Refused, read_topology
from server.insights.store import cell_catalog
from server.settings import settings

_WAIT_S = 600 # not a setting: a flag's default
Expand Down Expand Up @@ -324,7 +325,7 @@ async def writer_refusals(self) -> tuple[list[Refusal], Any, Any]:
topology = await read_topology(self.ch)
if refusals := topology.refusals():
return refusals, topology, {}
names = await cell_writers.host_names(self.ch, topology) if topology.multi else {}
names = await cell_catalog.host_names(self.ch, topology) if topology.multi else {}
allowed = cell_writers.Allowed(
users=frozenset(self.args.ch_allow_user),
user_directories=frozenset(self.args.ch_allow_user_directory),
Expand Down Expand Up @@ -406,7 +407,7 @@ async def begin(self) -> None:
# -- the scans ---------------------------------------------------------------------

async def scan(self, topology: Any, listed: cell_scan.TableList) -> cell_scan.ScanResult:
budget = cell_scan.Budget(self.args.scan_budget)
budget = cell_sync.Budget(self.args.scan_budget)
current, nxt = self.own_fp
return await cell_scan.scan(
self.ch, listed, topology, budget, cell_scan.configured_orgs(current, nxt)
Expand Down
2 changes: 1 addition & 1 deletion server/cli/cell_attach_beats.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
from typing import Any, Literal

from server.cli.cell_attach_pg import Wait
from server.cli.cell_ch import Refusal, Refused
from server.core import cell, instances
from server.core.cell_ch import Refusal, Refused

StoppedUnset = Literal["refuse", "wait"]

Expand Down
2 changes: 1 addition & 1 deletion server/cli/cell_attach_pg.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
from functools import partial
from typing import TYPE_CHECKING, Any

from server.cli.cell_ch import Refusal, Refused
from server.core import cell, cell_triggers, pg
from server.core.cell import ATTACH_LOCK_KEY
from server.core.cell_ch import Refusal, Refused

if TYPE_CHECKING:
from server.cli.cell_control import Owner
Expand Down
4 changes: 2 additions & 2 deletions server/cli/cell_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
from uuid import UUID

from server.cli import cell_attach_pg as apg
from server.cli.cell_ch import Refusal
from server.cli.cell_ch import Refused as Refused
from server.core import pg, retirement
from server.core.cell_ch import Refusal
from server.core.cell_ch import Refused as Refused
from server.router.directory import CONTROL_FIELDS, CONTROL_MAX_S
from server.settings import settings

Expand Down
2 changes: 1 addition & 1 deletion server/cli/cell_place.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
from server.cli import cell as cell_cli
from server.cli import cell_attach_pg as apg
from server.cli._ch import run_connected, writes
from server.cli.cell_ch import Refused
from server.cli.cell_control import Owner
from server.core import cell, pg
from server.core.cell_ch import Refused
from server.router.directory import PLACE_MAX_S
from server.settings import settings

Expand Down
155 changes: 20 additions & 135 deletions server/cli/cell_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,13 @@
import asyncio
import json
import re
import time
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any

from clickhouse_connect.driver.exceptions import DatabaseError

from server.cli.cell_ch import (
from server.core import cell_sync, ch_errors, ch_names, pg
from server.core.cell import ORG_FREE, named_org, org_name_refusal
from server.core.cell_ch import (
TAG,
WHOLE,
WHOLE_SQL,
Expand All @@ -42,14 +41,12 @@
per_host,
quote,
)
from server.cli.cell_writers import own_refusals
from server.core import ch_errors, ch_names, pg
from server.core.cell import ORG_FREE, named_org, org_name_refusal
from server.core.cell_triggers import ORG_TABLES
from server.core.ch import query_tag
from server.core.ch_errors import error_code, error_line
from server.erasure.store.census import classified_tables
from server.erasure.store.steps_pg import PG_CENSUS, PG_NO_MEMORY_CONTENT
from server.insights.store.cell_catalog import own_refusals

# Tables whose orgs are never placed: rows written before any check (a refused or
# unauthenticated request's log row keeps the org it named), and tables a view fills from
Expand Down Expand Up @@ -91,38 +88,11 @@
# How long past the budget a Postgres try may take before this side gives it up, so the
# server's own statement timeout fires first when the server answers.
_PG_MARGIN_S = 5.0 # not a setting: a margin over --scan-budget
_TIMEOUT_CODES = frozenset({159}) # TIMEOUT_EXCEEDED
_UNKNOWN_TABLE_CODES = frozenset({60, 81}) # UNKNOWN_TABLE, UNKNOWN_DATABASE

_sleep = asyncio.sleep # the seam the retry tests replace


class Budget:
"""`--scan-budget`: the seconds one full scan may take, shared by its statements."""

def __init__(self, seconds: float, clock: Callable[[], float] = time.monotonic) -> None:
self.seconds = float(seconds)
self._clock = clock
self._start = clock()

def elapsed(self) -> float:
return self._clock() - self._start

def left(self) -> float:
return self.seconds - self.elapsed()

def require(self, subject: str) -> float:
"""The seconds left, or Refused naming `subject` once none are."""
left = self.left()
if left <= 0:
raise Refused([_out_of_budget(subject, self.seconds)])
return left


def _out_of_budget(subject: str, seconds: float) -> Refusal:
return Refusal(subject, f"the scan ran out of --scan-budget ({seconds:g} s): raise it")


def pg_quote(name: str) -> str:
return '"' + name.replace('"', '""') + '"'

Expand Down Expand Up @@ -383,7 +353,7 @@ async def list_tables(
) -> TableList:
"""The scan's table list in both stores, with every refusal it found, attach's own
ClickHouse grants checked first; raises `Refused` when a host answers outside the
cluster's names. `names` is `cell_writers.host_names()` on a cluster of several hosts,
cluster's names. `names` is `cell_catalog.host_names()` on a cluster of several hosts,
else empty."""
with query_tag(TAG):
return await _list_tables(ch, pg_conn, topology, names)
Expand Down Expand Up @@ -501,7 +471,7 @@ def pg_skip_scan_sql(table: str) -> str:
return _PG_SKIP_SCAN.format(table=f"{pg_quote(schema)}.{pg_quote(name)}")


async def pg_orgs(table: str, budget: Budget) -> list[str]:
async def pg_orgs(table: str, budget: cell_sync.Budget) -> list[str]:
subject = f"postgres {table}"
while True:
left = budget.require(subject)
Expand Down Expand Up @@ -529,7 +499,7 @@ async def pg_orgs(table: str, budget: Budget) -> list[str]:
# budget ran out only when there is none left, since the timeout is what was left.
timed_out = getattr(failure, "sqlstate", None) == "57014" and budget.left() <= 1
if bound.expired() or timed_out:
raise Refused([_out_of_budget(subject, budget.seconds)]) from None
raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None
# A connect that finds Postgres down comes as PostgresUnavailableError.
if not (pg.is_outage(failure) or isinstance(failure, pg.PostgresUnavailableError)):
raise Refused([Refusal(subject, f"the scan failed: {failure}")]) from failure
Expand All @@ -545,63 +515,6 @@ def _ch_transient(e: BaseException) -> bool:
)


def _on_cluster(topology: Topology) -> str:
return f" ON CLUSTER {quote(topology.cluster)}" if topology.multi else ""


async def _system(ch: Any, sql: str, topology: Topology, budget: Budget, subject: str) -> None:
"""A SYSTEM statement bounded by what is left of `budget`. On several hosts it carries
`ON CLUSTER` (`_on_cluster`) and each host's status is read back: every listed host must
answer, each with status 0 (pinned with `distributed_ddl_output_mode = 'throw'`, so a
profile's `never_throw` hides none). One that does not finish in time refuses as the
budget when the budget is spent, else with the server's error and the hosts it did not
finish on: a replica that cannot sync."""
seconds = max(1, int(budget.require(subject)))
# `receive_timeout` bounds SYNC REPLICA's own wait.
bounds = {"max_execution_time": seconds, "receive_timeout": seconds}
what = sql.split(" ON CLUSTER ")[0].split(" `")[0]
try:
if not topology.multi:
await ch.execute(sql, settings=bounds)
return
body = await ch.raw_bytes(
sql,
settings={
**bounds,
"distributed_ddl_output_mode": "throw",
"distributed_ddl_task_timeout": seconds,
"default_format": "JSONEachRow",
},
)
rows = _body_rows(body.decode("utf-8", "replace").splitlines(), "status")
except Exception as e: # only a timeout is judged here; the rest pass to the caller
if error_code(e) not in _TIMEOUT_CODES:
raise
# The statement was given the budget's whole seconds, so up to one is left over.
if budget.left() <= 1:
raise Refused([_out_of_budget(subject, budget.seconds)]) from None
done = {str(r.get("host")) for r in getattr(e, "rows", ()) if str(r.get("status")) == "0"}
unfinished = ", ".join(sorted(topology.hosts - done)) or "this server"
raise Refused(
[Refusal(subject, f"{what} did not finish on {unfinished}: {error_line(e)}")]
) from e
failed = [r for r in rows if int(r["status"]) != 0]
if failed:
raise Refused(
[Refusal(f"replica {r.get('host')}", f"{subject}: {r.get('error')}") for r in failed]
)
answered = {str(r.get("host")) for r in rows}
if answered != topology.hosts:
missing = ", ".join(sorted(topology.hosts - answered)) or "none"
raise Refused(
[
Refusal(
subject, f"the hosts that answered are not the listed ones (missing: {missing})"
)
]
)


def ch_scan_sql(table: Table, topology: Topology) -> str:
"""The distinct orgs of `table` as hex (bytes: an org that is not UTF-8 is judged, never
decoded into another name); on several hosts, first a row naming the configured host
Expand All @@ -616,60 +529,32 @@ def ch_scan_sql(table: Table, topology: Topology) -> str:
return f"{host} UNION ALL {orgs} FORMAT JSONEachRow"


class BodyError(DatabaseError):
"""An error ClickHouse wrote into a response body after it began streaming rows: a
server error as the driver raises one, its text keeping the server's `Code: <n>.`, so it
is judged transient or not, and by code, as any other. `rows` are the rows before it."""

def __init__(self, text: str, rows: list[dict[str, Any]] | None = None) -> None:
super().__init__(text)
self.rows = list(rows or [])


def _body_rows(lines: Iterable[str], key: str) -> list[dict[str, Any]]:
"""The JSONEachRow rows of a body, each holding `key`; BodyError at the first line that
is not one (an `{"exception": …}` row, or the text tail a server appends once a
statement fails after its first block), with the rest of the body as its text."""
lines = list(lines)
rows: list[dict[str, Any]] = []
for i, line in enumerate(lines):
if not line.strip():
continue
try:
row = json.loads(line)
except ValueError:
row = None
if not isinstance(row, dict) or key not in row:
exception = row.get("exception") if isinstance(row, dict) else None
raise BodyError(str(exception or "\n".join(lines[i:]))[:2000], rows)
rows.append(row)
return rows


async def _read_scan(
ch: Any, sql: str, params: dict[str, Any], extra: dict[str, Any]
) -> list[dict[str, Any]]:
chunks: list[bytes] = []
async with ch.raw_stream(sql, params, settings=extra) as stream:
async for chunk in stream:
chunks.append(chunk)
return _body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k")
return cell_sync._body_rows(b"".join(chunks).decode("utf-8", "replace").splitlines(), "k")


async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) -> list[bytes | None]:
async def ch_orgs(
ch: Any, table: Table, topology: Topology, budget: cell_sync.Budget
) -> list[bytes | None]:
subject = f"clickhouse {table.canonical}"
while True:
budget.require(subject)
try:
# Rows buffered on any replica land before the table is read.
flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{_on_cluster(topology)}"
await _system(ch, flush, topology, budget, subject)
flush = f"SYSTEM FLUSH ASYNC INSERT QUEUE{cell_sync._on_cluster(topology)}"
await cell_sync._system(ch, flush, topology, budget, subject)
if table.engine.startswith("Replicated"):
# Every replica pulls every part acknowledged before now, so the read
# through any one of them sees them all.
await _system(
await cell_sync._system(
ch,
f"SYSTEM SYNC REPLICA{_on_cluster(topology)} {table.sql} LIGHTWEIGHT",
f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} {table.sql} LIGHTWEIGHT",
topology,
budget,
subject,
Expand All @@ -695,9 +580,9 @@ async def ch_orgs(ch: Any, table: Table, topology: Topology, budget: Budget) ->
if r["k"] == "o"
]
# A timeout is the budget only when none is left: a profile's estimate throws early.
if error_code(failure) in _TIMEOUT_CODES and budget.left() <= 1:
raise Refused([_out_of_budget(subject, budget.seconds)]) from None
if error_code(failure) in _TIMEOUT_CODES or not _ch_transient(failure):
if error_code(failure) in cell_sync._TIMEOUT_CODES and budget.left() <= 1:
raise Refused([cell_sync._out_of_budget(subject, budget.seconds)]) from None
if error_code(failure) in cell_sync._TIMEOUT_CODES or not _ch_transient(failure):
raise Refused(
[Refusal(subject, f"the scan failed: {error_line(failure)}")]
) from failure
Expand All @@ -722,7 +607,7 @@ async def scan(
ch: Any,
tables: TableList,
topology: Topology,
budget: Budget,
budget: cell_sync.Budget,
configured: Iterable[str] = (),
) -> ScanResult:
"""The distinct orgs of every table of `tables` and the `configured` orgs; Refused, with
Expand Down
2 changes: 1 addition & 1 deletion server/cli/cell_scim.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
from server.auth import claims, revocation, scim, scim_people, subject_revocation
from server.auth.scim_routes import scim_settings_problem
from server.cli import cell_control
from server.cli.cell_ch import WHOLE_SQL, Refusal, Refused
from server.core import cell, ch
from server.core.cell_ch import WHOLE_SQL, Refusal, Refused
from server.core.datetimes import as_aware_utc
from server.settings import settings

Expand Down
19 changes: 10 additions & 9 deletions server/cli/cell_update_participants.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@
from typing import Any, cast
from uuid import UUID

from server.cli import cell_control, cell_scan, cell_writers
from server.cli.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology
from server.core import cell, ch_names, instances
from server.cli import cell_control
from server.core import cell, cell_sync, ch_names, instances
from server.core.cell_ch import WHOLE_SQL, Refused, per_host, quote, read_topology
from server.core.ch import query_tag
from server.core.provider_process import (
certified,
has_certificate,
participant_tag,
proves_stopped,
)
from server.insights.store import cell_catalog
from server.insights.store import provider_participants as participants


Expand Down Expand Up @@ -88,8 +89,8 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]:
raise cell_control._refused("participant census cannot resolve the configured cluster")
if problems := topology.refusals():
raise Refused(problems)
names = await cell_writers.host_names(ch, topology) if topology.multi else {}
if problems := (await cell_writers.own_refusals(ch, topology, names))[0]:
names = await cell_catalog.host_names(ch, topology) if topology.multi else {}
if problems := (await cell_catalog.own_refusals(ch, topology, names))[0]:
raise Refused(problems)
database = ch_names.database()
tables = await per_host(
Expand Down Expand Up @@ -122,14 +123,14 @@ async def census(ch: Any) -> dict[str, dict[str, Any]]:
),
{"db": database},
)
if any(len(value) != 1 for value in rows.values()) or cell_writers.extra_replicas(
if any(len(value) != 1 for value in rows.values()) or cell_catalog.extra_replicas(
row for value in rows.values() for row in value
):
raise cell_control._refused("participant replica set includes an unverified replica")
budget = cell_scan.Budget(cell_control.TRANSACTION_S)
await cell_scan._system(
budget = cell_sync.Budget(cell_control.TRANSACTION_S)
await cell_sync._system(
ch,
f"SYSTEM SYNC REPLICA{cell_scan._on_cluster(topology)} "
f"SYSTEM SYNC REPLICA{cell_sync._on_cluster(topology)} "
f"{quote(database)}.`_provider_participants` LIGHTWEIGHT",
topology,
budget,
Expand Down
Loading