diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 64755d9039..696f9ce428 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -11,6 +11,7 @@ import shutil import signal import socket +import stat import subprocess import sys import time @@ -751,6 +752,99 @@ def dependency_receipt(root: Path, output: Path) -> dict: return {"path": str(output), "sha256": sha(output)} +def _progress_row(row, *, budget: bool) -> bool: + if type(row) is not list or not row or not all(type(n) is int and 0 <= n < 2**63 for n in row): + return False + if budget: + return ( + len(row) == 7 + and row[:2] == [0, 1] + and row[2] in (0, 1) + and row[4] <= 7020000 + and row[6] in (0, 1) + ) + if row[0] == 0: + return row == [0, 1, 100, 45500] + if row[0] == 1: + return ( + len(row) == 6 + and row[1] in (*range(1, 9), *range(10, 26)) + and row[2] <= 456 + and row[3] <= 45500 + ) + return len(row) == 4 and row[0] == 2 and row[1] <= 3216 and row[3] in (0, 1) + + +def _read_provenance_progress(path: Path, *, budget: bool): + try: + info = path.lstat() + except FileNotFoundError: + return None + require( + stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1, + "Owned regular provenance diagnostic", + ) + cap, records = (512, 2) if budget else (823552, 3217) + require(info.st_size <= cap, "Bounded provenance diagnostic") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + with os.fdopen(fd, "rb", closefd=False) as stream: + actual = os.fstat(stream.fileno()) + require( + (actual.st_dev, actual.st_ino, actual.st_uid, actual.st_nlink) + == (info.st_dev, info.st_ino, os.getuid(), 1) + and stat.S_ISREG(actual.st_mode), + "Stable provenance diagnostic", + ) + data = stream.read(cap + 1) + finally: + os.close(fd) + require(len(data) <= cap, "Bounded provenance diagnostic read") + incomplete = int(bool(data) and not data.endswith(b"\n")) + lines = data.splitlines() + require( + len(lines) <= records and all(len(line) + 1 <= 256 for line in lines), + "Bounded provenance diagnostic records", + ) + if incomplete: + lines = lines[:-1] + rows = [json.loads(line.decode("ascii")) for line in lines] + require(all(_progress_row(row, budget=budget) for row in rows), "Numeric provenance records") + if budget: + require(all(row[2] == index for index, row in enumerate(rows)), "Ordered budget snapshots") + elif rows: + require(rows[0] == [0, 1, 100, 45500], "Provenance diagnostic header") + require( + all(row[0] == 1 for row in rows[1:-1]) and (len(rows) == 1 or rows[-1][0] in (1, 2)), + "Ordered provenance records", + ) + if rows[-1][0] == 2: + require(rows[-1][1] == len(rows) - 1, "Provenance record count") + return rows, incomplete + + +def _publish_provenance_progress(fixture: Path, destination: Path) -> None: + diagnostic = None + try: + child = _read_provenance_progress(fixture / "provenance-progress.jsonl", budget=False) + budget = _read_provenance_progress(fixture / "provenance-budget.jsonl", budget=True) + if child is not None or budget is not None: + diagnostic = { + "schema": 1, + "rows": child[0] if child else [], + "budget": budget[0] if budget else [], + "incomplete_tail": int(bool((child and child[1]) or (budget and budget[1]))), + } + except (OSError, ValueError, RuntimeError): + diagnostic = {"schema": 1, "invalid": 1} + if diagnostic is not None: + try: + write(destination / "provenance-progress.json", diagnostic) + except OSError: + # Missing diagnostics cannot replace the required work or cleanup result. + pass + + def publish( folder: Path, destination: Path, @@ -775,6 +869,7 @@ def publish( for name in ("source-before.json", "source-after.json", "startup-diagnostics.json"): if (folder / name).is_file(): shutil.copyfile(folder / name, destination / name) + _publish_provenance_progress(fixture, destination) # Only the fixed numeric-only API diagnostic is public; raw API logs stay private. costs = fixture / "serving-cell/request-costs.jsonl" diagnostic = None diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 0eb1611ce0..58770abae1 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -393,7 +393,75 @@ def chain_update(chain, index, value): chain.update(raw) -async def verify_pages(conn, ch, context, workload, codec, native_hash): +class _ProofProgress: + """Optional numeric evidence; its storage never decides the proof outcome.""" + + def __init__(self, path, *, budget=False): + self.budget = budget + self.limit = 2 if budget else 3217 + self.emitted = self.dropped = self.bytes = self.page = self.verified = 0 + self.wall, self.cpu = time.perf_counter_ns(), time.process_time_ns() + self.stream = None + self.disabled = False + try: + self.stream = path.open("x", encoding="ascii") + os.fchmod(self.stream.fileno(), 0o600) + except OSError: + self.disabled = True + + def __enter__(self): + if not self.budget: + self.emit([0, 1, 100, 45500]) + return self + + def emit(self, row): + if self.disabled: + return + if not all(type(n) is int and 0 <= n < 2**63 for n in row): + self.disabled = True + return + data = json.dumps(row, separators=(",", ":")) + "\n" + reserve = int(not self.budget and row[0] != 2) + if ( + len(data) > 256 + or self.bytes + len(data) > 823552 + or self.emitted >= self.limit - reserve + ): + self.dropped += 1 + return + try: + self.stream.write(data) + self.stream.flush() + self.bytes += len(data) + self.emitted += 1 + except (OSError, ValueError): + self.disabled = True + + def mark(self, phase, page=None, verified=None): + if page is not None: + self.page, self.verified = page, verified + self.emit( + [ + 1, + phase, + self.page, + self.verified, + time.perf_counter_ns() - self.wall, + time.process_time_ns() - self.cpu, + ] + ) + + def __exit__(self, *exc): + if not self.budget: + self.emit([2, self.emitted, self.dropped, int(self.disabled)]) + if self.stream is not None: + try: + self.stream.close() + except (OSError, ValueError): + pass + + +async def verify_pages(conn, ch, context, workload, codec, native_hash, *, progress=None): db = database(context["databases"]["ch_schema"]) seen_seq = set() seen_hash = set() @@ -402,10 +470,26 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): observed_chain = hashlib.sha256() authority_chain = hashlib.sha256() raw_keys = hashlib.sha256() - for page in workload.pages(): + page_number = 1 + verified = 0 + + def mark(phase): + if progress is not None: + progress(phase, page_number, verified) + + def pages(): + nonlocal page_number + mark(1) # Input iteration includes the existing stream's final trailer checks. + for page in workload.pages(): + yield page + page_number += 1 + mark(1) + + for page in pages(): need(0 < len(page) <= 100, "fixed proof page") ids = [item["memory_id"] for item in page] complete_ids(ids, ids) + mark(2) # PG page query. fetched = await conn.fetch( pg_page_sql(), context["org"], @@ -413,6 +497,7 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): [uuid.UUID(value) for value in ids], len(page) + 1, ) + mark(3) # PG decoding, comparisons and independent hash/chain checks. need(len(fetched) == len(page), "PG page exact cardinality/overflow") values = [pg_record(row, PG_FIELDS) for row in fetched] complete_ids([identifier(row["c_memory_id"]) for row in values], ids) @@ -468,6 +553,7 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): chain_update(raw_keys, count, dict(head=sorted(head), revision=sorted(revision))) count += 1 fields = codec.p["memory_fields"] + mark(4) # CH memory query. data = await ch_result( ch, "SELECT " @@ -485,11 +571,13 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): fields, len(page) + 1, ) + mark(5) # CH memory comparison. complete_ids([identifier(row[fields.index("memory_id")]) for row in data], ids) for value in data: row = codec.row(dict(zip(fields, value, strict=True)), origin="ch", comparison=True) equal(row, checked[row["memory_id"]]) fields = codec.p["audit_fields"] + mark(6) # CH audit query. data = await ch_result( ch, "SELECT " @@ -504,10 +592,13 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): fields, len(page) + 1, ) + mark(7) # CH audit comparison: only its successful completion verifies a page. complete_ids([hex64(row[fields.index("row_hash")], native=True) for row in data], hashes) for value in data: audit = codec.audit(dict(zip(fields, value, strict=True)), origin="ch") equal(audit, audits[audit["row_hash"]]) + verified += len(page) + mark(8) # Complete workload counts and chain equality. need(count == len(seen_seq) == len(seen_hash) == 45500, "complete exact workload identities") need( expected_chain.hexdigest() == observed_chain.hexdigest(), @@ -544,33 +635,50 @@ def codec_contract(): async def independent_provenance(context, output): + with _ProofProgress(output.with_name("provenance-progress.jsonl")) as progress: + await _independent_provenance(context, output, progress) + + +async def _independent_provenance(context, output, progress): from server.observability.audit import row_matches_hash + progress.mark(10) # Codec contract. codec = codec_contract() + progress.mark(11) # Bound serving-process receipt. context = dict( context, _api_server=read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] ) first = None summary = None try: + progress.mark(12) # Input/token/receipt validation. with Workload(context, codec) as workload: + progress.mark(13) # Connection entry. async with connections(context) as (conn, ch): try: + progress.mark(14) # First PG snapshot. page = await begin_observation(conn, context, "repeatable read") + progress.mark(15) # Initial CH census. initial_ch = await ch_census(ch, context) + progress.mark(16) # Enter the unchanged page proof. checked = await verify_pages( - conn, ch, context, workload, codec, row_matches_hash + conn, ch, context, workload, codec, row_matches_hash, progress=progress.mark ) + progress.mark(17) # Close page snapshot. await end_observation(conn, page) + progress.mark(18) # Final PG snapshot. final = await begin_observation( conn, context, "read committed", backend=page["backend_pid"] ) + progress.mark(19) # Close final PG snapshot. await end_observation(conn, final) need( conn.is_in_transaction() is False, "final CH checks outside both PG snapshots", ) + progress.mark(20) # Final CH census. final_ch = await ch_census(ch, context) + progress.mark(21) # Construct the unchanged proof receipt. summary = dict( schema=2, fixture_id=context["fixture_id"], @@ -603,14 +711,120 @@ async def independent_provenance(context, output): first = error if first is None else first if first is not None: raise first + progress.mark(22) # Close connections, only after a successful proof. + progress.mark(23) # Close input streams without overwriting a failing phase. except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. first = error if first is None else first if first is not None: raise first + progress.mark(24) need(summary is not None, "complete proof before closure") + progress.mark(25) # Final proof receipt write. write(output, summary) +def _observe_request_phase(function, stage, sample_var, origin, emit, tag, *, asynchronous): + from contextlib import contextmanager + from functools import wraps + + @contextmanager + def observation(sample): + started = cpu = None + try: + started, cpu = time.perf_counter_ns(), time.process_time_ns() + except Exception: # noqa: BLE001 — optional clocks cannot prevent the original call. + pass + try: + yield + finally: + failed = int(sys.exc_info()[0] is not None) + if started is not None and cpu is not None: + try: + # _begin_request binds its request tag inside the original call. + current = tag() + correlation = ( + int.from_bytes(hashlib.sha256(current.encode()).digest()) if current else 0 + ) + emit( + [ + 1, + sample["ordinal"], + stage, + started - origin, + time.perf_counter_ns() - started, + time.process_time_ns() - cpu, + correlation, + 0, + failed, + sample["done"], + 0, + ] + ) + except Exception: # noqa: BLE001 — optional diagnostics cannot replace a result or failure. + pass + + if asynchronous: + + @wraps(function) + async def async_phase(*args, **kwargs): + sample = sample_var.get() + if sample is None: + return await function(*args, **kwargs) + with observation(sample): + return await function(*args, **kwargs) + + return async_phase + + @wraps(function) + def sync_phase(*args, **kwargs): + sample = sample_var.get() + if sample is None: + return function(*args, **kwargs) + with observation(sample): + return function(*args, **kwargs) + + return sync_phase + + +def _observe_http_phases(http, sample_var, origin, emit, tag): + # These are existing endpoint boundaries, not replacement admission decisions. + names = ( + "_begin_request", + "_extract_auth", + "_read_body", + "_authorize", + "_bind_context_and_policy", + "_idempotency_gate", + "_start_telemetry", + "_run_handler", + "_finish_success", + ) + saved = [] + + def restore(): + for name, original in reversed(saved): + setattr(http, name, original) + + try: + for stage, name in enumerate(names, start=10): + original = getattr(http, name) + observed = _observe_request_phase( + original, + stage, + sample_var, + origin, + emit, + tag, + asynchronous=name != "_begin_request", + ) + setattr(http, name, observed) + saved.append((name, original)) + except BaseException: # Partial instrumentation restores before propagating setup failure. + restore() + raise + return restore + + def api(context: dict, output: Path) -> None: import uvicorn @@ -714,6 +928,7 @@ def emit_cost(row): emit_cost([0, 1, *samples]) original_traced = otel.traced + restore_http_phases = None def measured_traced(name, **attrs): sample = sample_var.get() @@ -750,9 +965,21 @@ def measured_span(): return measured_span() def measured_application(): + nonlocal restore_http_phases # Uvicorn still imports the normal app on its running loop, before lifespan. from server.main import app + if restore_http_phases is None: + from importlib import import_module + + restore_http_phases = _observe_http_phases( + import_module("server.transport.http"), + sample_var, + origin, + emit_cost, + current_query_tag, + ) + async def measured(scope, receive, send): nonlocal ordinal if scope.get("type") != "http": @@ -821,6 +1048,8 @@ async def measured_send(message): reload=False, ) finally: + if restore_http_phases is not None: + restore_http_phases() otel.traced = original_traced emit_cost([3, ordinal, emitted, dropped, int(disabled)]) if stream is not None: @@ -2410,7 +2639,32 @@ def ownership(self) -> dict: } def provenance(self) -> None: - self.action("provenance", settings=self.cli_settings) + started = time.monotonic() + returned = 0 + with _ProofProgress(self.w / "provenance-budget.jsonl", budget=True) as progress: + + def snapshot(terminal): + now = time.monotonic() + elapsed = max(0, int((now - self.o.started) * 1000)) + progress.emit( + [ + 0, + 1, + terminal, + elapsed, + max(0, 7020000 - elapsed), + max(0, int((now - started) * 1000)), + returned, + ] + ) + + snapshot(0) + try: + self.action("provenance", settings=self.cli_settings) + returned = 1 + finally: + # This is the observed action exit, not the exact monitor refusal instant. + snapshot(1) def block_teardown(self, error: BaseException) -> None: # Retain the mutation barrier even if the negative receipt cannot be created. diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index feaa6d9015..00fabf467e 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -1083,6 +1083,47 @@ def delegate(name, **attrs): def invoke(mode): state["mode"] = mode pending = [] + + def begin(op, request, response): + return "request-id" + + async def extract(request): + return marker + + async def read_body(op, request, request_id, auth): + return marker + + async def authorize(op, response, call, payload): + return marker + + async def policy(op, request, call, target, payload): + return marker + + async def replay(op, request, response, call, target, payload): + return marker + + async def telemetry(op, response, call): + return marker + + async def handler(op, payload, call): + return marker + + async def finish(op, call, payload, result, elapsed_ms, idem): + return marker + + originals = dict( + _begin_request=begin, + _extract_auth=extract, + _read_body=read_body, + _authorize=authorize, + _bind_context_and_policy=policy, + _idempotency_gate=replay, + _start_telemetry=telemetry, + _run_handler=handler, + _finish_success=finish, + ) + http = SimpleNamespace(**originals) + monkeypatch.setitem(sys.modules, "server.transport.http", http) receive_message = {"type": "http.request", "body": b"private-request-value"} async def receive(): @@ -1108,6 +1149,25 @@ async def application(scope, actual_receive, actual_send): index = state["calls"] ch.set_query_tag(f"memory.set request-{index}") selected = index in samples + if selected and mode == "phases": + assert http._begin_request(marker, marker, marker) == "request-id" + assert await http._extract_auth(marker) is marker + assert await http._read_body(marker, marker, marker, marker) is marker + assert await http._authorize(marker, marker, marker, marker) is marker + assert ( + await http._bind_context_and_policy(marker, marker, marker, marker, marker) + is marker + ) + assert ( + await http._idempotency_gate(marker, marker, marker, marker, marker, marker) + is marker + ) + assert await http._start_telemetry(marker, marker, marker) is marker + assert await http._run_handler(marker, marker, marker) is marker + assert ( + await http._finish_success(marker, marker, marker, marker, marker, marker) + is marker + ) count = 225 if mode == "cap" and index == samples[0] else 1 for _ in range(count): with otel.traced( @@ -1143,6 +1203,10 @@ async def child(): async def exercise(factory): app = factory() + installed = {name: getattr(http, name) for name in originals} + assert all(installed[name] is not original for name, original in originals.items()) + factory() + assert all(getattr(http, name) is installed[name] for name in originals) state["release"] = asyncio.Event() try: for scope in ( @@ -1169,6 +1233,9 @@ def run(application_factory, **options): "workers": 1, "reload": False, } + if mode == "startup-error": + application_factory() + raise error asyncio.run(exercise(application_factory)) monkeypatch.setattr(uvicorn, "run", run) @@ -1202,6 +1269,7 @@ def open_path(path, *args, **kwargs): except BaseException as caught: # noqa: BLE001 — compare the exact fake delegate's failure, including cancellation state["error"] = caught assert otel.traced is delegate + assert vars(http) == originals path = folder / "request-costs.jsonl" state["raw"] = path.read_text() if path.exists() else "" state["rows"] = [json.loads(line) for line in state["raw"].splitlines()] diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) diff --git a/tests/unit/test_cell_trigger_provenance_progress.py b/tests/unit/test_cell_trigger_provenance_progress.py new file mode 100644 index 0000000000..6eae7bebee --- /dev/null +++ b/tests/unit/test_cell_trigger_provenance_progress.py @@ -0,0 +1,369 @@ +"""Proof diagnostics retain the interrupted phase without changing the outcome.""" + +import asyncio +import json +from contextlib import asynccontextmanager, contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader + + +@pytest.mark.parametrize("mode", ["refused", "cancelled", "diagnostic-io-error"]) +async def test_provenance_records_the_active_phase_and_preserves_the_first_error( + keeper_config_case, monkeypatch, tmp_path, mode +): + native, _, _, _, _ = keeper_config_case + folder = tmp_path / "serving-cell" + folder.mkdir() + native.write(folder / "launch.json", {"server": {"pid": 7}}) + expected = asyncio.CancelledError() if mode == "cancelled" else RuntimeError("private-proof") + calls = [] + + @contextmanager + def workload(*args): + calls.append("workload") + try: + yield SimpleNamespace() + finally: + calls.append("workload-close") + + @asynccontextmanager + async def connections(context): + calls.append("connect") + try: + yield SimpleNamespace(is_in_transaction=lambda: False), object() + finally: + calls.append("disconnect") + + async def begin(*args): + calls.append("begin") + return {"backend_pid": 7} + + async def census(*args): + calls.append("census") + return {} + + async def verify(*args, progress=None): + calls.append("verify") + if progress is not None: + progress(2, 1, 0) + raise expected + + monkeypatch.setattr(native, "codec_contract", lambda: object()) + monkeypatch.setattr(native, "Workload", workload) + monkeypatch.setattr(native, "connections", connections) + monkeypatch.setattr(native, "begin_observation", begin) + monkeypatch.setattr(native, "ch_census", census) + monkeypatch.setattr(native, "verify_pages", verify) + if mode == "diagnostic-io-error": + original_open = Path.open + + def open_path(path, *args, **kwargs): + if path.name == "provenance-progress.jsonl": + raise OSError("private-diagnostic-storage") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", open_path) + with pytest.raises(BaseException) as caught: + await native.independent_provenance( + {"workdir": str(tmp_path)}, tmp_path / "provenance.json" + ) + assert caught.value is expected + assert calls == [ + "workload", + "connect", + "begin", + "census", + "verify", + "disconnect", + "workload-close", + ] + assert not (tmp_path / "provenance.json").exists() + progress = tmp_path / "provenance-progress.jsonl" + if mode == "diagnostic-io-error": + assert not progress.exists() + else: + raw = progress.read_text() + rows = [json.loads(line) for line in raw.splitlines()] + assert rows[0] == [0, 1, 100, 45500] + assert [row for row in rows if row[0] == 1][-1][1:4] == [2, 1, 0] + assert all(type(n) is int and n >= 0 for row in rows for n in row) + assert "private" not in raw and "pid" not in raw + + +@pytest.mark.parametrize("mode", ["returned", "refused", "cancelled"]) +def test_owner_budget_is_recorded_before_cleanup_without_replacing_the_result( + keeper_config_case, monkeypatch, mode +): + native, instance, owner, _, _ = keeper_config_case + clock = SimpleNamespace(now=6990.0) + monkeypatch.setattr( + native, + "time", + SimpleNamespace( + monotonic=lambda: clock.now, + perf_counter_ns=lambda: 17, + process_time_ns=lambda: 11, + ), + ) + owner.started = 0.0 + instance.cli_settings = {"same": "private-settings"} + expected = asyncio.CancelledError() if mode == "cancelled" else RuntimeError("private-refusal") + calls = [] + + def action(task, **kwargs): + calls.append((task, kwargs)) + clock.now = 7021.0 + if mode != "returned": + raise expected + + monkeypatch.setattr(instance, "action", action) + if mode == "returned": + assert instance.provenance() is None + else: + with pytest.raises(BaseException) as caught: + instance.provenance() + assert caught.value is expected + assert calls == [("provenance", {"settings": instance.cli_settings})] + raw = (instance.w / "provenance-budget.jsonl").read_text() + rows = [json.loads(line) for line in raw.splitlines()] + assert rows == [ + [0, 1, 0, 6990000, 30000, 0, 0], + [0, 1, 1, 7021000, 0, 31000, int(mode == "returned")], + ] + assert "private" not in raw + + +def test_progress_publication_is_separate_from_the_acceptance_result(tmp_path): + from scripts.gates import cell_trigger_acceptance as creator + + folder = tmp_path / "private" + source = folder / "fixture" + source.mkdir(parents=True) + child = [[0, 1, 100, 45500], [1, 2, 1, 0, 5, 3], [2, 2, 0, 0]] + budget = [[0, 1, 0, 6990000, 30000, 0, 0], [0, 1, 1, 7021000, 0, 31000, 0]] + for name, rows in [("provenance-progress", child), ("provenance-budget", budget)]: + (source / (name + ".jsonl")).write_text("".join(json.dumps(row) + "\n" for row in rows)) + (source / "private.log").write_text("private-settings-token") + destination = tmp_path / "published" + creator.publish( + folder, + destination, + RuntimeError("private-error"), + phase="native", + diagnostic_error_type=None, + ) + result = json.loads((destination / "result.json").read_text()) + assert result["accepted"] is False and result["error_type"] == "RuntimeError" + progress = json.loads((destination / "provenance-progress.json").read_text()) + assert progress == {"schema": 1, "rows": child, "budget": budget, "incomplete_tail": 0} + assert {f.name for f in destination.iterdir()} == {"result.json", "provenance-progress.json"} + assert "private-" not in "".join(f.read_text() for f in destination.iterdir()) + + +@pytest.mark.parametrize( + "fault,phase", [("", 1), ("native_hash", 3), ("ch_value", 5), ("audit_missing", 7)] +) +async def test_page_progress_follows_the_full_existing_hash_and_comparison_checks( + keeper_config_case, protocol_reader, monkeypatch, fault, phase +): + from tests.unit.test_cell_trigger_provenance_reuse import _exercise + + native, *_ = keeper_config_case + original = native.verify_pages + records = [] + + async def observed(*args, **kwargs): + return await original(*args, **kwargs, progress=lambda *row: records.append(row)) + + monkeypatch.setattr(native, "verify_pages", observed) + await _exercise(native, monkeypatch, 2, False, fault) + assert records[0] == (1, 1, 0) + assert records[-1] == ((1, 2, 2) if not fault else (phase, 1, 0)) + if not fault: + assert records == [(number, 1, 0) for number in range(1, 8)] + [(1, 2, 2)] + + +@pytest.mark.parametrize("phase", [2, 4, 6]) +async def test_page_cancellation_keeps_the_interrupted_query_phase( + keeper_config_case, protocol_reader, monkeypatch, phase +): + from tests.unit.test_cell_trigger_provenance_reuse import _exercise + + native, *_ = keeper_config_case + original = native.verify_pages + records = [] + expected = asyncio.CancelledError() + + async def observed(conn, ch, *args, **kwargs): + fetch, query = conn.fetch, ch.query + + async def fetch_page(*call): + if phase == 2: + raise expected + return await fetch(*call) + + async def query_page(sql, **call): + if phase == (6 if ".audit_log " in sql else 4): + raise expected + return await query(sql, **call) + + conn.fetch, ch.query = fetch_page, query_page + return await original(conn, ch, *args, **kwargs, progress=lambda *row: records.append(row)) + + monkeypatch.setattr(native, "verify_pages", observed) + with pytest.raises(asyncio.CancelledError) as caught: + await _exercise(native, monkeypatch, 2, False) + assert caught.value is expected + assert records[-1] == (phase, 1, 0) + + +@pytest.mark.parametrize( + "mode", + [ + "string", + "bool", + "negative", + "phase", + "rows", + "clock", + "bytes", + "records", + "symlink", + "read-error", + "write-error", + "truncated", + "absent", + ], +) +def test_progress_export_is_closed_bounded_and_never_replaces_acceptance( + tmp_path, monkeypatch, mode +): + from scripts.gates import cell_trigger_acceptance as creator + + folder = tmp_path / "private" + source = folder / "fixture/provenance-progress.jsonl" + source.parent.mkdir(parents=True) + rows = [[0, 1, 100, 45500], [1, 2, 1, 0, 5, 3]] + if mode == "string": + rows[1][1] = "private-token" + elif mode == "bool": + rows[1][2] = True + elif mode == "negative": + rows[1][4] = -1 + elif mode == "phase": + rows[1][1] = 26 + elif mode == "rows": + rows[1][3] = 45501 + elif mode == "clock": + rows[1][5] = 2**63 + elif mode == "records": + rows += [rows[1]] * 3216 + contents = "".join(json.dumps(row) + "\n" for row in rows) + if mode == "bytes": + contents = " " * 823553 + elif mode == "truncated": + contents += '[1,"private-unfinished' + if mode == "symlink": + target = folder / "private-source" + target.write_text("private-token") + source.symlink_to(target) + elif mode != "absent": + source.write_text(contents) + if mode == "read-error": + original_open = creator.os.open + + def open_path(path, *args, **kwargs): + if path == source: + raise OSError("private-read-error") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(creator.os, "open", open_path) + if mode == "write-error": + original_write = creator.write + + def write(path, *args, **kwargs): + if path.name == "provenance-progress.json": + raise OSError("private-write-error") + return original_write(path, *args, **kwargs) + + monkeypatch.setattr(creator, "write", write) + destination = tmp_path / "published" + creator.publish(folder, destination, None, phase="native", diagnostic_error_type=None) + result = json.loads((destination / "result.json").read_text()) + assert result["accepted"] is True and result["error_type"] == "" + if mode in {"absent", "write-error"}: + assert {p.name for p in destination.iterdir()} == {"result.json"} + else: + diagnostic = json.loads((destination / "provenance-progress.json").read_text()) + assert diagnostic == ( + {"schema": 1, "rows": rows, "budget": [], "incomplete_tail": 1} + if mode == "truncated" + else {"schema": 1, "invalid": 1} + ) + assert "private-" not in "".join(p.read_text() for p in destination.iterdir()) + + +def test_progress_stream_caps_records_and_keeps_its_numeric_drop_count(keeper_config_case): + native, instance, *_ = keeper_config_case + path = instance.w / "bounded-progress.jsonl" + with native._ProofProgress(path) as progress: + for _ in range(4000): + progress.mark(2, 1, 0) + raw = path.read_bytes() + rows = [json.loads(line) for line in raw.splitlines()] + assert len(rows) == 3217 and len(raw) <= 823552 + assert rows[-1] == [2, 3216, 785, 0] + assert all(len(line) + 1 <= 256 for line in raw.splitlines()) + assert path.stat().st_mode & 0o077 == 0 + + +@pytest.mark.parametrize("operation", ["write", "flush", "close"]) +def test_progress_storage_errors_preserve_the_active_work_exception( + keeper_config_case, monkeypatch, operation +): + native, instance, *_ = keeper_config_case + original_open = Path.open + path = instance.w / "failed-progress.jsonl" + opened = [] + + class Stream: + def __init__(self, actual): + self.actual = actual + + def __getattr__(self, name): + return getattr(self.actual, name) + + def write(self, value): + if operation == "write": + raise OSError("private-write") + return self.actual.write(value) + + def flush(self): + if operation == "flush": + raise OSError("private-flush") + return self.actual.flush() + + def close(self): + self.actual.close() + if operation == "close": + raise OSError("private-close") + + def open_path(target, *args, **kwargs): + stream = original_open(target, *args, **kwargs) + if target == path: + opened.append(stream) + return Stream(stream) + return stream + + monkeypatch.setattr(Path, "open", open_path) + original = RuntimeError("private-proof") + with pytest.raises(RuntimeError) as caught: + with native._ProofProgress(path) as progress: + progress.mark(2, 1, 0) + raise original + assert caught.value is original and opened[0].closed diff --git a/tests/unit/test_cell_trigger_request_phases.py b/tests/unit/test_cell_trigger_request_phases.py new file mode 100644 index 0000000000..973a0a4e9a --- /dev/null +++ b/tests/unit/test_cell_trigger_request_phases.py @@ -0,0 +1,207 @@ +"""Selected request phases preserve the native delegate and its failures.""" + +import asyncio +import hashlib +import inspect +from contextvars import ContextVar +from types import SimpleNamespace + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import request_cost_api_case as request_cost_api_case + + +@pytest.mark.parametrize("asynchronous", [True, False]) +async def test_selected_phase_preserves_result_arguments_signature_and_bound_tag( + keeper_config_case, monkeypatch, asynchronous +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + rows, seen = [], [] + state = {"tag": "private-previous-tag"} + marker = object() + + def delegate(value, *, option): + seen.append((value, option)) + state["tag"] = "memory.set private-current-request" + return marker + + async def async_delegate(value, *, option): + return delegate(value, option=option) + + original = async_delegate if asynchronous else delegate + monkeypatch.setattr(native.time, "perf_counter_ns", lambda: 120) + monkeypatch.setattr(native.time, "process_time_ns", lambda: 70) + wrapped = native._observe_request_phase( + original, 10, sample, 100, rows.append, lambda: state["tag"], asynchronous=asynchronous + ) + result = ( + await wrapped(marker, option=marker) if asynchronous else wrapped(marker, option=marker) + ) + assert result is marker and seen == [(marker, marker)] + assert inspect.signature(wrapped) == inspect.signature(original) + expected = int.from_bytes(hashlib.sha256(state["tag"].encode()).digest()) + assert rows == [[1, 8192, 10, 20, 0, 0, expected, 0, 0, 0, 0]] + assert "private" not in str(rows) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +async def test_unselected_phase_touches_no_clock_tag_or_writer( + keeper_config_case, monkeypatch, asynchronous +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + marker = object() + + def forbidden(*args): + raise AssertionError("unselected diagnostic ran") + + def original(): + return marker + + async def async_original(): + return marker + + monkeypatch.setattr(native.time, "perf_counter_ns", forbidden) + monkeypatch.setattr(native.time, "process_time_ns", forbidden) + wrapped = native._observe_request_phase( + async_original if asynchronous else original, + 11, + sample, + 0, + forbidden, + forbidden, + asynchronous=asynchronous, + ) + assert (await wrapped() if asynchronous else wrapped()) is marker + + +@pytest.mark.parametrize("error", [RuntimeError("delegate"), asyncio.CancelledError()]) +async def test_original_async_failure_survives_diagnostic_storage_failure( + keeper_config_case, error +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + + async def original(): + raise error + + def broken_writer(row): + assert row[8] == 1 + raise OSError("diagnostic disk") + + wrapped = native._observe_request_phase( + original, 12, sample, 0, broken_writer, lambda: "", asynchronous=True + ) + with pytest.raises(type(error)) as caught: + await wrapped() + assert caught.value is error + + +def test_sync_failure_survives_unencodable_optional_tag(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + error = ValueError("delegate") + + def original(): + raise error + + wrapped = native._observe_request_phase( + original, 10, sample, 0, lambda row: None, lambda: "\ud800", asynchronous=False + ) + with pytest.raises(ValueError) as caught: + wrapped() + assert caught.value is error + + +def test_installed_http_boundaries_are_exact_and_restored(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + + def begin(): + return object() + + async def phase(): + return object() + + names = ( + "_begin_request", + "_extract_auth", + "_read_body", + "_authorize", + "_bind_context_and_policy", + "_idempotency_gate", + "_start_telemetry", + "_run_handler", + "_finish_success", + ) + originals = {name: begin if name == "_begin_request" else phase for name in names} + http = SimpleNamespace(**originals) + restore = native._observe_http_phases(http, sample, 0, lambda row: None, lambda: "") + for name in names: + assert getattr(http, name) is not originals[name] + assert inspect.iscoroutinefunction(getattr(http, name)) == (name != "_begin_request") + restore() + assert vars(http) == originals + + +def test_partial_installation_restores_before_raising(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + + def begin(*args): + return None + + async def extract(*args): + return None + + originals = {"_begin_request": begin, "_extract_auth": extract} + http = SimpleNamespace(**originals) + with pytest.raises(AttributeError): + native._observe_http_phases(http, sample, 0, lambda row: None, lambda: "") + assert vars(http) == originals + + +def test_optional_clock_failure_keeps_the_original_call(keeper_config_case, monkeypatch): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + marker = object() + calls, rows = [], [] + + def original(): + calls.append(1) + return marker + + def broken_clock(): + raise OSError("optional clock") + + monkeypatch.setattr(native.time, "perf_counter_ns", broken_clock) + wrapped = native._observe_request_phase( + original, 10, sample, 0, rows.append, lambda: "", asynchronous=False + ) + assert wrapped() is marker and calls == [1] and rows == [] + + +def test_native_phase_installation_restores_after_server_startup_error(request_cost_api_case): + state = request_cost_api_case("startup-error") + assert state["error"] is state["expected_error"] + assert state["calls"] == 0 and state["rows"][-1] == [3, 0, 0, 0, 0] + + +def test_native_api_records_all_nine_boundaries_only_for_existing_samples(request_cost_api_case): + state = request_cost_api_case("phases") + assert state["error"] is None + rows = state["rows"] + phases = [row for row in rows if row[0] == 1 and row[2] >= 10] + assert len(phases) == 54 + for ordinal, digest in state["tags"].items(): + selected = [row for row in phases if row[1] == ordinal] + assert [row[2] for row in selected] == list(range(10, 19)) + assert all(row[6] == digest and row[8:] == [0, 0, 0] for row in selected) + assert rows[-1] == [3, 18432, 72, 0, 0] + assert state["calls"] == 18432 and state["sends"] == 2 * 18432 diff --git a/tests/unit/test_code_graph_stats_pass_budget.py b/tests/unit/test_code_graph_stats_pass_budget.py index a7e194e2f7..cd39781c4d 100644 --- a/tests/unit/test_code_graph_stats_pass_budget.py +++ b/tests/unit/test_code_graph_stats_pass_budget.py @@ -22,6 +22,7 @@ from typing import Any import pytest +import pytest_asyncio from server.code.api.models_status import CodeIndexStatusOutput from server.code.app import index_status @@ -607,6 +608,8 @@ async def test_a_cancelled_leg_issues_no_further_statements( assert repo.rows("code_unit_risk") == [] +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_pass_deadline_clock") async def test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -836,3 +839,101 @@ async def test_graph_stats_state_shows_unit_risk_progress() -> None: ) == "" ) + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.25]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_pass_deadline_clock") +async def test_pass_deadline_fixture_ignores_external_scheduler_elapsed(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + actual_pass = cgs.run_pass + actual_create_task = asyncio.create_task + owned = [] + + def monotonic(): + return real_monotonic() + elapsed + + async def run_pass(): + nonlocal elapsed + elapsed += scheduler_s + await asyncio.sleep(0.001) + return await actual_pass() + + def create_task(coro, *args, **kwargs): + task = actual_create_task(coro, *args, **kwargs) + owned.append(task) + return task + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cgs, "run_pass", run_pass) + monkeypatch.setattr(asyncio, "create_task", create_task) + try: + await test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count( + monkeypatch + ) + finally: + for task in owned: + if not task.done(): + task.cancel() + await asyncio.gather(*owned, return_exceptions=True) + + +@pytest_asyncio.fixture(loop_scope="function") +async def _pass_deadline_clock(monkeypatch): + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + async def to_thread(func, /, *args, **kwargs): + return func(*args, **kwargs) + + # Real pure calculations stay on this private clock instead of an executor. + monkeypatch.setattr(asyncio, "to_thread", to_thread) + # The cancellation protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cgs, "time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "graph-stats deadline leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_pass_deadline_clock_cancels_a_genuinely_blocked_query( + monkeypatch, _pass_deadline_clock +): + loop = asyncio.get_running_loop() + clock = _pass_deadline_clock + assert loop.time.__self__ is clock and cgs.time is clock + actual_edges = _Repo.edges_for_symbols + cancelled = [] + + async def edges(repo, *args, **kwargs): + started = loop.time() + try: + return await actual_edges(repo, *args, **kwargs) + except asyncio.CancelledError: + cancelled.append(loop.time() - started) + raise + + monkeypatch.setattr(_Repo, "edges_for_symbols", edges) + started = loop.time() + await test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count(monkeypatch) + assert cancelled and all(elapsed == pytest.approx(0.2) for elapsed in cancelled) + assert loop.time() - started == pytest.approx(0.25) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75