From 4bda1f1e9807063b58fb6ce0e53717f6dcddab34 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 00:17:57 +0000 Subject: [PATCH 1/7] Record bounded Cell provenance progress An interrupted proof needs its last observed page and validation phase. Numeric checkpoints retain that evidence while preserving proof checks, deadlines and the original failure. Signed-off-by: zlareb1 (cherry picked from commit f43e6d139e241dbf61697930d35bb82790f98c0e) Signed-off-by: zlareb1 --- scripts/gates/cell_trigger_acceptance.py | 95 +++++ .../gates/testsupport/cell_trigger/native.py | 145 ++++++- .../test_cell_trigger_provenance_progress.py | 369 ++++++++++++++++++ 3 files changed, 605 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_cell_trigger_provenance_progress.py diff --git a/scripts/gates/cell_trigger_acceptance.py b/scripts/gates/cell_trigger_acceptance.py index 64755d9039..696f9ce428 100644 --- a/scripts/gates/cell_trigger_acceptance.py +++ b/scripts/gates/cell_trigger_acceptance.py @@ -11,6 +11,7 @@ import shutil import signal import socket +import stat import subprocess import sys import time @@ -751,6 +752,99 @@ def dependency_receipt(root: Path, output: Path) -> dict: return {"path": str(output), "sha256": sha(output)} +def _progress_row(row, *, budget: bool) -> bool: + if type(row) is not list or not row or not all(type(n) is int and 0 <= n < 2**63 for n in row): + return False + if budget: + return ( + len(row) == 7 + and row[:2] == [0, 1] + and row[2] in (0, 1) + and row[4] <= 7020000 + and row[6] in (0, 1) + ) + if row[0] == 0: + return row == [0, 1, 100, 45500] + if row[0] == 1: + return ( + len(row) == 6 + and row[1] in (*range(1, 9), *range(10, 26)) + and row[2] <= 456 + and row[3] <= 45500 + ) + return len(row) == 4 and row[0] == 2 and row[1] <= 3216 and row[3] in (0, 1) + + +def _read_provenance_progress(path: Path, *, budget: bool): + try: + info = path.lstat() + except FileNotFoundError: + return None + require( + stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() and info.st_nlink == 1, + "Owned regular provenance diagnostic", + ) + cap, records = (512, 2) if budget else (823552, 3217) + require(info.st_size <= cap, "Bounded provenance diagnostic") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + with os.fdopen(fd, "rb", closefd=False) as stream: + actual = os.fstat(stream.fileno()) + require( + (actual.st_dev, actual.st_ino, actual.st_uid, actual.st_nlink) + == (info.st_dev, info.st_ino, os.getuid(), 1) + and stat.S_ISREG(actual.st_mode), + "Stable provenance diagnostic", + ) + data = stream.read(cap + 1) + finally: + os.close(fd) + require(len(data) <= cap, "Bounded provenance diagnostic read") + incomplete = int(bool(data) and not data.endswith(b"\n")) + lines = data.splitlines() + require( + len(lines) <= records and all(len(line) + 1 <= 256 for line in lines), + "Bounded provenance diagnostic records", + ) + if incomplete: + lines = lines[:-1] + rows = [json.loads(line.decode("ascii")) for line in lines] + require(all(_progress_row(row, budget=budget) for row in rows), "Numeric provenance records") + if budget: + require(all(row[2] == index for index, row in enumerate(rows)), "Ordered budget snapshots") + elif rows: + require(rows[0] == [0, 1, 100, 45500], "Provenance diagnostic header") + require( + all(row[0] == 1 for row in rows[1:-1]) and (len(rows) == 1 or rows[-1][0] in (1, 2)), + "Ordered provenance records", + ) + if rows[-1][0] == 2: + require(rows[-1][1] == len(rows) - 1, "Provenance record count") + return rows, incomplete + + +def _publish_provenance_progress(fixture: Path, destination: Path) -> None: + diagnostic = None + try: + child = _read_provenance_progress(fixture / "provenance-progress.jsonl", budget=False) + budget = _read_provenance_progress(fixture / "provenance-budget.jsonl", budget=True) + if child is not None or budget is not None: + diagnostic = { + "schema": 1, + "rows": child[0] if child else [], + "budget": budget[0] if budget else [], + "incomplete_tail": int(bool((child and child[1]) or (budget and budget[1]))), + } + except (OSError, ValueError, RuntimeError): + diagnostic = {"schema": 1, "invalid": 1} + if diagnostic is not None: + try: + write(destination / "provenance-progress.json", diagnostic) + except OSError: + # Missing diagnostics cannot replace the required work or cleanup result. + pass + + def publish( folder: Path, destination: Path, @@ -775,6 +869,7 @@ def publish( for name in ("source-before.json", "source-after.json", "startup-diagnostics.json"): if (folder / name).is_file(): shutil.copyfile(folder / name, destination / name) + _publish_provenance_progress(fixture, destination) # Only the fixed numeric-only API diagnostic is public; raw API logs stay private. costs = fixture / "serving-cell/request-costs.jsonl" diagnostic = None diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index 0eb1611ce0..ece22c7d07 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -393,7 +393,75 @@ def chain_update(chain, index, value): chain.update(raw) -async def verify_pages(conn, ch, context, workload, codec, native_hash): +class _ProofProgress: + """Optional numeric evidence; its storage never decides the proof outcome.""" + + def __init__(self, path, *, budget=False): + self.budget = budget + self.limit = 2 if budget else 3217 + self.emitted = self.dropped = self.bytes = self.page = self.verified = 0 + self.wall, self.cpu = time.perf_counter_ns(), time.process_time_ns() + self.stream = None + self.disabled = False + try: + self.stream = path.open("x", encoding="ascii") + os.fchmod(self.stream.fileno(), 0o600) + except OSError: + self.disabled = True + + def __enter__(self): + if not self.budget: + self.emit([0, 1, 100, 45500]) + return self + + def emit(self, row): + if self.disabled: + return + if not all(type(n) is int and 0 <= n < 2**63 for n in row): + self.disabled = True + return + data = json.dumps(row, separators=(",", ":")) + "\n" + reserve = int(not self.budget and row[0] != 2) + if ( + len(data) > 256 + or self.bytes + len(data) > 823552 + or self.emitted >= self.limit - reserve + ): + self.dropped += 1 + return + try: + self.stream.write(data) + self.stream.flush() + self.bytes += len(data) + self.emitted += 1 + except (OSError, ValueError): + self.disabled = True + + def mark(self, phase, page=None, verified=None): + if page is not None: + self.page, self.verified = page, verified + self.emit( + [ + 1, + phase, + self.page, + self.verified, + time.perf_counter_ns() - self.wall, + time.process_time_ns() - self.cpu, + ] + ) + + def __exit__(self, *exc): + if not self.budget: + self.emit([2, self.emitted, self.dropped, int(self.disabled)]) + if self.stream is not None: + try: + self.stream.close() + except (OSError, ValueError): + pass + + +async def verify_pages(conn, ch, context, workload, codec, native_hash, *, progress=None): db = database(context["databases"]["ch_schema"]) seen_seq = set() seen_hash = set() @@ -402,10 +470,26 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): observed_chain = hashlib.sha256() authority_chain = hashlib.sha256() raw_keys = hashlib.sha256() - for page in workload.pages(): + page_number = 1 + verified = 0 + + def mark(phase): + if progress is not None: + progress(phase, page_number, verified) + + def pages(): + nonlocal page_number + mark(1) # Input iteration includes the existing stream's final trailer checks. + for page in workload.pages(): + yield page + page_number += 1 + mark(1) + + for page in pages(): need(0 < len(page) <= 100, "fixed proof page") ids = [item["memory_id"] for item in page] complete_ids(ids, ids) + mark(2) # PG page query. fetched = await conn.fetch( pg_page_sql(), context["org"], @@ -413,6 +497,7 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): [uuid.UUID(value) for value in ids], len(page) + 1, ) + mark(3) # PG decoding, comparisons and independent hash/chain checks. need(len(fetched) == len(page), "PG page exact cardinality/overflow") values = [pg_record(row, PG_FIELDS) for row in fetched] complete_ids([identifier(row["c_memory_id"]) for row in values], ids) @@ -468,6 +553,7 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): chain_update(raw_keys, count, dict(head=sorted(head), revision=sorted(revision))) count += 1 fields = codec.p["memory_fields"] + mark(4) # CH memory query. data = await ch_result( ch, "SELECT " @@ -485,11 +571,13 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): fields, len(page) + 1, ) + mark(5) # CH memory comparison. complete_ids([identifier(row[fields.index("memory_id")]) for row in data], ids) for value in data: row = codec.row(dict(zip(fields, value, strict=True)), origin="ch", comparison=True) equal(row, checked[row["memory_id"]]) fields = codec.p["audit_fields"] + mark(6) # CH audit query. data = await ch_result( ch, "SELECT " @@ -504,10 +592,13 @@ async def verify_pages(conn, ch, context, workload, codec, native_hash): fields, len(page) + 1, ) + mark(7) # CH audit comparison: only its successful completion verifies a page. complete_ids([hex64(row[fields.index("row_hash")], native=True) for row in data], hashes) for value in data: audit = codec.audit(dict(zip(fields, value, strict=True)), origin="ch") equal(audit, audits[audit["row_hash"]]) + verified += len(page) + mark(8) # Complete workload counts and chain equality. need(count == len(seen_seq) == len(seen_hash) == 45500, "complete exact workload identities") need( expected_chain.hexdigest() == observed_chain.hexdigest(), @@ -544,33 +635,50 @@ def codec_contract(): async def independent_provenance(context, output): + with _ProofProgress(output.with_name("provenance-progress.jsonl")) as progress: + await _independent_provenance(context, output, progress) + + +async def _independent_provenance(context, output, progress): from server.observability.audit import row_matches_hash + progress.mark(10) # Codec contract. codec = codec_contract() + progress.mark(11) # Bound serving-process receipt. context = dict( context, _api_server=read(Path(context["workdir"]) / "serving-cell/launch.json")["server"] ) first = None summary = None try: + progress.mark(12) # Input/token/receipt validation. with Workload(context, codec) as workload: + progress.mark(13) # Connection entry. async with connections(context) as (conn, ch): try: + progress.mark(14) # First PG snapshot. page = await begin_observation(conn, context, "repeatable read") + progress.mark(15) # Initial CH census. initial_ch = await ch_census(ch, context) + progress.mark(16) # Enter the unchanged page proof. checked = await verify_pages( - conn, ch, context, workload, codec, row_matches_hash + conn, ch, context, workload, codec, row_matches_hash, progress=progress.mark ) + progress.mark(17) # Close page snapshot. await end_observation(conn, page) + progress.mark(18) # Final PG snapshot. final = await begin_observation( conn, context, "read committed", backend=page["backend_pid"] ) + progress.mark(19) # Close final PG snapshot. await end_observation(conn, final) need( conn.is_in_transaction() is False, "final CH checks outside both PG snapshots", ) + progress.mark(20) # Final CH census. final_ch = await ch_census(ch, context) + progress.mark(21) # Construct the unchanged proof receipt. summary = dict( schema=2, fixture_id=context["fixture_id"], @@ -603,11 +711,15 @@ async def independent_provenance(context, output): first = error if first is None else first if first is not None: raise first + progress.mark(22) # Close connections, only after a successful proof. + progress.mark(23) # Close input streams without overwriting a failing phase. except BaseException as error: # noqa: BLE001 — owned cleanup must finish while retaining the first failure. first = error if first is None else first if first is not None: raise first + progress.mark(24) need(summary is not None, "complete proof before closure") + progress.mark(25) # Final proof receipt write. write(output, summary) @@ -2410,7 +2522,32 @@ def ownership(self) -> dict: } def provenance(self) -> None: - self.action("provenance", settings=self.cli_settings) + started = time.monotonic() + returned = 0 + with _ProofProgress(self.w / "provenance-budget.jsonl", budget=True) as progress: + + def snapshot(terminal): + now = time.monotonic() + elapsed = max(0, int((now - self.o.started) * 1000)) + progress.emit( + [ + 0, + 1, + terminal, + elapsed, + max(0, 7020000 - elapsed), + max(0, int((now - started) * 1000)), + returned, + ] + ) + + snapshot(0) + try: + self.action("provenance", settings=self.cli_settings) + returned = 1 + finally: + # This is the observed action exit, not the exact monitor refusal instant. + snapshot(1) def block_teardown(self, error: BaseException) -> None: # Retain the mutation barrier even if the negative receipt cannot be created. diff --git a/tests/unit/test_cell_trigger_provenance_progress.py b/tests/unit/test_cell_trigger_provenance_progress.py new file mode 100644 index 0000000000..6eae7bebee --- /dev/null +++ b/tests/unit/test_cell_trigger_provenance_progress.py @@ -0,0 +1,369 @@ +"""Proof diagnostics retain the interrupted phase without changing the outcome.""" + +import asyncio +import json +from contextlib import asynccontextmanager, contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader + + +@pytest.mark.parametrize("mode", ["refused", "cancelled", "diagnostic-io-error"]) +async def test_provenance_records_the_active_phase_and_preserves_the_first_error( + keeper_config_case, monkeypatch, tmp_path, mode +): + native, _, _, _, _ = keeper_config_case + folder = tmp_path / "serving-cell" + folder.mkdir() + native.write(folder / "launch.json", {"server": {"pid": 7}}) + expected = asyncio.CancelledError() if mode == "cancelled" else RuntimeError("private-proof") + calls = [] + + @contextmanager + def workload(*args): + calls.append("workload") + try: + yield SimpleNamespace() + finally: + calls.append("workload-close") + + @asynccontextmanager + async def connections(context): + calls.append("connect") + try: + yield SimpleNamespace(is_in_transaction=lambda: False), object() + finally: + calls.append("disconnect") + + async def begin(*args): + calls.append("begin") + return {"backend_pid": 7} + + async def census(*args): + calls.append("census") + return {} + + async def verify(*args, progress=None): + calls.append("verify") + if progress is not None: + progress(2, 1, 0) + raise expected + + monkeypatch.setattr(native, "codec_contract", lambda: object()) + monkeypatch.setattr(native, "Workload", workload) + monkeypatch.setattr(native, "connections", connections) + monkeypatch.setattr(native, "begin_observation", begin) + monkeypatch.setattr(native, "ch_census", census) + monkeypatch.setattr(native, "verify_pages", verify) + if mode == "diagnostic-io-error": + original_open = Path.open + + def open_path(path, *args, **kwargs): + if path.name == "provenance-progress.jsonl": + raise OSError("private-diagnostic-storage") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", open_path) + with pytest.raises(BaseException) as caught: + await native.independent_provenance( + {"workdir": str(tmp_path)}, tmp_path / "provenance.json" + ) + assert caught.value is expected + assert calls == [ + "workload", + "connect", + "begin", + "census", + "verify", + "disconnect", + "workload-close", + ] + assert not (tmp_path / "provenance.json").exists() + progress = tmp_path / "provenance-progress.jsonl" + if mode == "diagnostic-io-error": + assert not progress.exists() + else: + raw = progress.read_text() + rows = [json.loads(line) for line in raw.splitlines()] + assert rows[0] == [0, 1, 100, 45500] + assert [row for row in rows if row[0] == 1][-1][1:4] == [2, 1, 0] + assert all(type(n) is int and n >= 0 for row in rows for n in row) + assert "private" not in raw and "pid" not in raw + + +@pytest.mark.parametrize("mode", ["returned", "refused", "cancelled"]) +def test_owner_budget_is_recorded_before_cleanup_without_replacing_the_result( + keeper_config_case, monkeypatch, mode +): + native, instance, owner, _, _ = keeper_config_case + clock = SimpleNamespace(now=6990.0) + monkeypatch.setattr( + native, + "time", + SimpleNamespace( + monotonic=lambda: clock.now, + perf_counter_ns=lambda: 17, + process_time_ns=lambda: 11, + ), + ) + owner.started = 0.0 + instance.cli_settings = {"same": "private-settings"} + expected = asyncio.CancelledError() if mode == "cancelled" else RuntimeError("private-refusal") + calls = [] + + def action(task, **kwargs): + calls.append((task, kwargs)) + clock.now = 7021.0 + if mode != "returned": + raise expected + + monkeypatch.setattr(instance, "action", action) + if mode == "returned": + assert instance.provenance() is None + else: + with pytest.raises(BaseException) as caught: + instance.provenance() + assert caught.value is expected + assert calls == [("provenance", {"settings": instance.cli_settings})] + raw = (instance.w / "provenance-budget.jsonl").read_text() + rows = [json.loads(line) for line in raw.splitlines()] + assert rows == [ + [0, 1, 0, 6990000, 30000, 0, 0], + [0, 1, 1, 7021000, 0, 31000, int(mode == "returned")], + ] + assert "private" not in raw + + +def test_progress_publication_is_separate_from_the_acceptance_result(tmp_path): + from scripts.gates import cell_trigger_acceptance as creator + + folder = tmp_path / "private" + source = folder / "fixture" + source.mkdir(parents=True) + child = [[0, 1, 100, 45500], [1, 2, 1, 0, 5, 3], [2, 2, 0, 0]] + budget = [[0, 1, 0, 6990000, 30000, 0, 0], [0, 1, 1, 7021000, 0, 31000, 0]] + for name, rows in [("provenance-progress", child), ("provenance-budget", budget)]: + (source / (name + ".jsonl")).write_text("".join(json.dumps(row) + "\n" for row in rows)) + (source / "private.log").write_text("private-settings-token") + destination = tmp_path / "published" + creator.publish( + folder, + destination, + RuntimeError("private-error"), + phase="native", + diagnostic_error_type=None, + ) + result = json.loads((destination / "result.json").read_text()) + assert result["accepted"] is False and result["error_type"] == "RuntimeError" + progress = json.loads((destination / "provenance-progress.json").read_text()) + assert progress == {"schema": 1, "rows": child, "budget": budget, "incomplete_tail": 0} + assert {f.name for f in destination.iterdir()} == {"result.json", "provenance-progress.json"} + assert "private-" not in "".join(f.read_text() for f in destination.iterdir()) + + +@pytest.mark.parametrize( + "fault,phase", [("", 1), ("native_hash", 3), ("ch_value", 5), ("audit_missing", 7)] +) +async def test_page_progress_follows_the_full_existing_hash_and_comparison_checks( + keeper_config_case, protocol_reader, monkeypatch, fault, phase +): + from tests.unit.test_cell_trigger_provenance_reuse import _exercise + + native, *_ = keeper_config_case + original = native.verify_pages + records = [] + + async def observed(*args, **kwargs): + return await original(*args, **kwargs, progress=lambda *row: records.append(row)) + + monkeypatch.setattr(native, "verify_pages", observed) + await _exercise(native, monkeypatch, 2, False, fault) + assert records[0] == (1, 1, 0) + assert records[-1] == ((1, 2, 2) if not fault else (phase, 1, 0)) + if not fault: + assert records == [(number, 1, 0) for number in range(1, 8)] + [(1, 2, 2)] + + +@pytest.mark.parametrize("phase", [2, 4, 6]) +async def test_page_cancellation_keeps_the_interrupted_query_phase( + keeper_config_case, protocol_reader, monkeypatch, phase +): + from tests.unit.test_cell_trigger_provenance_reuse import _exercise + + native, *_ = keeper_config_case + original = native.verify_pages + records = [] + expected = asyncio.CancelledError() + + async def observed(conn, ch, *args, **kwargs): + fetch, query = conn.fetch, ch.query + + async def fetch_page(*call): + if phase == 2: + raise expected + return await fetch(*call) + + async def query_page(sql, **call): + if phase == (6 if ".audit_log " in sql else 4): + raise expected + return await query(sql, **call) + + conn.fetch, ch.query = fetch_page, query_page + return await original(conn, ch, *args, **kwargs, progress=lambda *row: records.append(row)) + + monkeypatch.setattr(native, "verify_pages", observed) + with pytest.raises(asyncio.CancelledError) as caught: + await _exercise(native, monkeypatch, 2, False) + assert caught.value is expected + assert records[-1] == (phase, 1, 0) + + +@pytest.mark.parametrize( + "mode", + [ + "string", + "bool", + "negative", + "phase", + "rows", + "clock", + "bytes", + "records", + "symlink", + "read-error", + "write-error", + "truncated", + "absent", + ], +) +def test_progress_export_is_closed_bounded_and_never_replaces_acceptance( + tmp_path, monkeypatch, mode +): + from scripts.gates import cell_trigger_acceptance as creator + + folder = tmp_path / "private" + source = folder / "fixture/provenance-progress.jsonl" + source.parent.mkdir(parents=True) + rows = [[0, 1, 100, 45500], [1, 2, 1, 0, 5, 3]] + if mode == "string": + rows[1][1] = "private-token" + elif mode == "bool": + rows[1][2] = True + elif mode == "negative": + rows[1][4] = -1 + elif mode == "phase": + rows[1][1] = 26 + elif mode == "rows": + rows[1][3] = 45501 + elif mode == "clock": + rows[1][5] = 2**63 + elif mode == "records": + rows += [rows[1]] * 3216 + contents = "".join(json.dumps(row) + "\n" for row in rows) + if mode == "bytes": + contents = " " * 823553 + elif mode == "truncated": + contents += '[1,"private-unfinished' + if mode == "symlink": + target = folder / "private-source" + target.write_text("private-token") + source.symlink_to(target) + elif mode != "absent": + source.write_text(contents) + if mode == "read-error": + original_open = creator.os.open + + def open_path(path, *args, **kwargs): + if path == source: + raise OSError("private-read-error") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(creator.os, "open", open_path) + if mode == "write-error": + original_write = creator.write + + def write(path, *args, **kwargs): + if path.name == "provenance-progress.json": + raise OSError("private-write-error") + return original_write(path, *args, **kwargs) + + monkeypatch.setattr(creator, "write", write) + destination = tmp_path / "published" + creator.publish(folder, destination, None, phase="native", diagnostic_error_type=None) + result = json.loads((destination / "result.json").read_text()) + assert result["accepted"] is True and result["error_type"] == "" + if mode in {"absent", "write-error"}: + assert {p.name for p in destination.iterdir()} == {"result.json"} + else: + diagnostic = json.loads((destination / "provenance-progress.json").read_text()) + assert diagnostic == ( + {"schema": 1, "rows": rows, "budget": [], "incomplete_tail": 1} + if mode == "truncated" + else {"schema": 1, "invalid": 1} + ) + assert "private-" not in "".join(p.read_text() for p in destination.iterdir()) + + +def test_progress_stream_caps_records_and_keeps_its_numeric_drop_count(keeper_config_case): + native, instance, *_ = keeper_config_case + path = instance.w / "bounded-progress.jsonl" + with native._ProofProgress(path) as progress: + for _ in range(4000): + progress.mark(2, 1, 0) + raw = path.read_bytes() + rows = [json.loads(line) for line in raw.splitlines()] + assert len(rows) == 3217 and len(raw) <= 823552 + assert rows[-1] == [2, 3216, 785, 0] + assert all(len(line) + 1 <= 256 for line in raw.splitlines()) + assert path.stat().st_mode & 0o077 == 0 + + +@pytest.mark.parametrize("operation", ["write", "flush", "close"]) +def test_progress_storage_errors_preserve_the_active_work_exception( + keeper_config_case, monkeypatch, operation +): + native, instance, *_ = keeper_config_case + original_open = Path.open + path = instance.w / "failed-progress.jsonl" + opened = [] + + class Stream: + def __init__(self, actual): + self.actual = actual + + def __getattr__(self, name): + return getattr(self.actual, name) + + def write(self, value): + if operation == "write": + raise OSError("private-write") + return self.actual.write(value) + + def flush(self): + if operation == "flush": + raise OSError("private-flush") + return self.actual.flush() + + def close(self): + self.actual.close() + if operation == "close": + raise OSError("private-close") + + def open_path(target, *args, **kwargs): + stream = original_open(target, *args, **kwargs) + if target == path: + opened.append(stream) + return Stream(stream) + return stream + + monkeypatch.setattr(Path, "open", open_path) + original = RuntimeError("private-proof") + with pytest.raises(RuntimeError) as caught: + with native._ProofProgress(path) as progress: + progress.mark(2, 1, 0) + raise original + assert caught.value is original and opened[0].closed From b3f974164247741408f8ecb225b09d5896fa4980 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 02:53:42 +0000 Subject: [PATCH 2/7] Record selected Cell request phases Bound HTTP helper timings to the existing six request samples so acceptance failures retain evidence for work outside named spans. Preserve delegation, failure identity and the original workload and limits. Signed-off-by: zlareb1 (cherry picked from commit fc936c8f4a646e770a6ed96de5542ba1abf96edf) Signed-off-by: zlareb1 --- .../gates/testsupport/cell_trigger/native.py | 117 ++++++++++ tests/unit/test_cell_trigger_ci_creator.py | 68 ++++++ .../unit/test_cell_trigger_request_phases.py | 207 ++++++++++++++++++ 3 files changed, 392 insertions(+) create mode 100644 tests/unit/test_cell_trigger_request_phases.py diff --git a/scripts/gates/testsupport/cell_trigger/native.py b/scripts/gates/testsupport/cell_trigger/native.py index ece22c7d07..58770abae1 100644 --- a/scripts/gates/testsupport/cell_trigger/native.py +++ b/scripts/gates/testsupport/cell_trigger/native.py @@ -723,6 +723,108 @@ async def _independent_provenance(context, output, progress): write(output, summary) +def _observe_request_phase(function, stage, sample_var, origin, emit, tag, *, asynchronous): + from contextlib import contextmanager + from functools import wraps + + @contextmanager + def observation(sample): + started = cpu = None + try: + started, cpu = time.perf_counter_ns(), time.process_time_ns() + except Exception: # noqa: BLE001 — optional clocks cannot prevent the original call. + pass + try: + yield + finally: + failed = int(sys.exc_info()[0] is not None) + if started is not None and cpu is not None: + try: + # _begin_request binds its request tag inside the original call. + current = tag() + correlation = ( + int.from_bytes(hashlib.sha256(current.encode()).digest()) if current else 0 + ) + emit( + [ + 1, + sample["ordinal"], + stage, + started - origin, + time.perf_counter_ns() - started, + time.process_time_ns() - cpu, + correlation, + 0, + failed, + sample["done"], + 0, + ] + ) + except Exception: # noqa: BLE001 — optional diagnostics cannot replace a result or failure. + pass + + if asynchronous: + + @wraps(function) + async def async_phase(*args, **kwargs): + sample = sample_var.get() + if sample is None: + return await function(*args, **kwargs) + with observation(sample): + return await function(*args, **kwargs) + + return async_phase + + @wraps(function) + def sync_phase(*args, **kwargs): + sample = sample_var.get() + if sample is None: + return function(*args, **kwargs) + with observation(sample): + return function(*args, **kwargs) + + return sync_phase + + +def _observe_http_phases(http, sample_var, origin, emit, tag): + # These are existing endpoint boundaries, not replacement admission decisions. + names = ( + "_begin_request", + "_extract_auth", + "_read_body", + "_authorize", + "_bind_context_and_policy", + "_idempotency_gate", + "_start_telemetry", + "_run_handler", + "_finish_success", + ) + saved = [] + + def restore(): + for name, original in reversed(saved): + setattr(http, name, original) + + try: + for stage, name in enumerate(names, start=10): + original = getattr(http, name) + observed = _observe_request_phase( + original, + stage, + sample_var, + origin, + emit, + tag, + asynchronous=name != "_begin_request", + ) + setattr(http, name, observed) + saved.append((name, original)) + except BaseException: # Partial instrumentation restores before propagating setup failure. + restore() + raise + return restore + + def api(context: dict, output: Path) -> None: import uvicorn @@ -826,6 +928,7 @@ def emit_cost(row): emit_cost([0, 1, *samples]) original_traced = otel.traced + restore_http_phases = None def measured_traced(name, **attrs): sample = sample_var.get() @@ -862,9 +965,21 @@ def measured_span(): return measured_span() def measured_application(): + nonlocal restore_http_phases # Uvicorn still imports the normal app on its running loop, before lifespan. from server.main import app + if restore_http_phases is None: + from importlib import import_module + + restore_http_phases = _observe_http_phases( + import_module("server.transport.http"), + sample_var, + origin, + emit_cost, + current_query_tag, + ) + async def measured(scope, receive, send): nonlocal ordinal if scope.get("type") != "http": @@ -933,6 +1048,8 @@ async def measured_send(message): reload=False, ) finally: + if restore_http_phases is not None: + restore_http_phases() otel.traced = original_traced emit_cost([3, ordinal, emitted, dropped, int(disabled)]) if stream is not None: diff --git a/tests/unit/test_cell_trigger_ci_creator.py b/tests/unit/test_cell_trigger_ci_creator.py index feaa6d9015..00fabf467e 100644 --- a/tests/unit/test_cell_trigger_ci_creator.py +++ b/tests/unit/test_cell_trigger_ci_creator.py @@ -1083,6 +1083,47 @@ def delegate(name, **attrs): def invoke(mode): state["mode"] = mode pending = [] + + def begin(op, request, response): + return "request-id" + + async def extract(request): + return marker + + async def read_body(op, request, request_id, auth): + return marker + + async def authorize(op, response, call, payload): + return marker + + async def policy(op, request, call, target, payload): + return marker + + async def replay(op, request, response, call, target, payload): + return marker + + async def telemetry(op, response, call): + return marker + + async def handler(op, payload, call): + return marker + + async def finish(op, call, payload, result, elapsed_ms, idem): + return marker + + originals = dict( + _begin_request=begin, + _extract_auth=extract, + _read_body=read_body, + _authorize=authorize, + _bind_context_and_policy=policy, + _idempotency_gate=replay, + _start_telemetry=telemetry, + _run_handler=handler, + _finish_success=finish, + ) + http = SimpleNamespace(**originals) + monkeypatch.setitem(sys.modules, "server.transport.http", http) receive_message = {"type": "http.request", "body": b"private-request-value"} async def receive(): @@ -1108,6 +1149,25 @@ async def application(scope, actual_receive, actual_send): index = state["calls"] ch.set_query_tag(f"memory.set request-{index}") selected = index in samples + if selected and mode == "phases": + assert http._begin_request(marker, marker, marker) == "request-id" + assert await http._extract_auth(marker) is marker + assert await http._read_body(marker, marker, marker, marker) is marker + assert await http._authorize(marker, marker, marker, marker) is marker + assert ( + await http._bind_context_and_policy(marker, marker, marker, marker, marker) + is marker + ) + assert ( + await http._idempotency_gate(marker, marker, marker, marker, marker, marker) + is marker + ) + assert await http._start_telemetry(marker, marker, marker) is marker + assert await http._run_handler(marker, marker, marker) is marker + assert ( + await http._finish_success(marker, marker, marker, marker, marker, marker) + is marker + ) count = 225 if mode == "cap" and index == samples[0] else 1 for _ in range(count): with otel.traced( @@ -1143,6 +1203,10 @@ async def child(): async def exercise(factory): app = factory() + installed = {name: getattr(http, name) for name in originals} + assert all(installed[name] is not original for name, original in originals.items()) + factory() + assert all(getattr(http, name) is installed[name] for name in originals) state["release"] = asyncio.Event() try: for scope in ( @@ -1169,6 +1233,9 @@ def run(application_factory, **options): "workers": 1, "reload": False, } + if mode == "startup-error": + application_factory() + raise error asyncio.run(exercise(application_factory)) monkeypatch.setattr(uvicorn, "run", run) @@ -1202,6 +1269,7 @@ def open_path(path, *args, **kwargs): except BaseException as caught: # noqa: BLE001 — compare the exact fake delegate's failure, including cancellation state["error"] = caught assert otel.traced is delegate + assert vars(http) == originals path = folder / "request-costs.jsonl" state["raw"] = path.read_text() if path.exists() else "" state["rows"] = [json.loads(line) for line in state["raw"].splitlines()] diff --git a/tests/unit/test_cell_trigger_request_phases.py b/tests/unit/test_cell_trigger_request_phases.py new file mode 100644 index 0000000000..973a0a4e9a --- /dev/null +++ b/tests/unit/test_cell_trigger_request_phases.py @@ -0,0 +1,207 @@ +"""Selected request phases preserve the native delegate and its failures.""" + +import asyncio +import hashlib +import inspect +from contextvars import ContextVar +from types import SimpleNamespace + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import request_cost_api_case as request_cost_api_case + + +@pytest.mark.parametrize("asynchronous", [True, False]) +async def test_selected_phase_preserves_result_arguments_signature_and_bound_tag( + keeper_config_case, monkeypatch, asynchronous +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + rows, seen = [], [] + state = {"tag": "private-previous-tag"} + marker = object() + + def delegate(value, *, option): + seen.append((value, option)) + state["tag"] = "memory.set private-current-request" + return marker + + async def async_delegate(value, *, option): + return delegate(value, option=option) + + original = async_delegate if asynchronous else delegate + monkeypatch.setattr(native.time, "perf_counter_ns", lambda: 120) + monkeypatch.setattr(native.time, "process_time_ns", lambda: 70) + wrapped = native._observe_request_phase( + original, 10, sample, 100, rows.append, lambda: state["tag"], asynchronous=asynchronous + ) + result = ( + await wrapped(marker, option=marker) if asynchronous else wrapped(marker, option=marker) + ) + assert result is marker and seen == [(marker, marker)] + assert inspect.signature(wrapped) == inspect.signature(original) + expected = int.from_bytes(hashlib.sha256(state["tag"].encode()).digest()) + assert rows == [[1, 8192, 10, 20, 0, 0, expected, 0, 0, 0, 0]] + assert "private" not in str(rows) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +async def test_unselected_phase_touches_no_clock_tag_or_writer( + keeper_config_case, monkeypatch, asynchronous +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + marker = object() + + def forbidden(*args): + raise AssertionError("unselected diagnostic ran") + + def original(): + return marker + + async def async_original(): + return marker + + monkeypatch.setattr(native.time, "perf_counter_ns", forbidden) + monkeypatch.setattr(native.time, "process_time_ns", forbidden) + wrapped = native._observe_request_phase( + async_original if asynchronous else original, + 11, + sample, + 0, + forbidden, + forbidden, + asynchronous=asynchronous, + ) + assert (await wrapped() if asynchronous else wrapped()) is marker + + +@pytest.mark.parametrize("error", [RuntimeError("delegate"), asyncio.CancelledError()]) +async def test_original_async_failure_survives_diagnostic_storage_failure( + keeper_config_case, error +): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + + async def original(): + raise error + + def broken_writer(row): + assert row[8] == 1 + raise OSError("diagnostic disk") + + wrapped = native._observe_request_phase( + original, 12, sample, 0, broken_writer, lambda: "", asynchronous=True + ) + with pytest.raises(type(error)) as caught: + await wrapped() + assert caught.value is error + + +def test_sync_failure_survives_unencodable_optional_tag(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + error = ValueError("delegate") + + def original(): + raise error + + wrapped = native._observe_request_phase( + original, 10, sample, 0, lambda row: None, lambda: "\ud800", asynchronous=False + ) + with pytest.raises(ValueError) as caught: + wrapped() + assert caught.value is error + + +def test_installed_http_boundaries_are_exact_and_restored(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + + def begin(): + return object() + + async def phase(): + return object() + + names = ( + "_begin_request", + "_extract_auth", + "_read_body", + "_authorize", + "_bind_context_and_policy", + "_idempotency_gate", + "_start_telemetry", + "_run_handler", + "_finish_success", + ) + originals = {name: begin if name == "_begin_request" else phase for name in names} + http = SimpleNamespace(**originals) + restore = native._observe_http_phases(http, sample, 0, lambda row: None, lambda: "") + for name in names: + assert getattr(http, name) is not originals[name] + assert inspect.iscoroutinefunction(getattr(http, name)) == (name != "_begin_request") + restore() + assert vars(http) == originals + + +def test_partial_installation_restores_before_raising(keeper_config_case): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + + def begin(*args): + return None + + async def extract(*args): + return None + + originals = {"_begin_request": begin, "_extract_auth": extract} + http = SimpleNamespace(**originals) + with pytest.raises(AttributeError): + native._observe_http_phases(http, sample, 0, lambda row: None, lambda: "") + assert vars(http) == originals + + +def test_optional_clock_failure_keeps_the_original_call(keeper_config_case, monkeypatch): + native, _, _, _, _ = keeper_config_case + sample = ContextVar("sample", default=None) + sample.set({"ordinal": 8192, "done": 0}) + marker = object() + calls, rows = [], [] + + def original(): + calls.append(1) + return marker + + def broken_clock(): + raise OSError("optional clock") + + monkeypatch.setattr(native.time, "perf_counter_ns", broken_clock) + wrapped = native._observe_request_phase( + original, 10, sample, 0, rows.append, lambda: "", asynchronous=False + ) + assert wrapped() is marker and calls == [1] and rows == [] + + +def test_native_phase_installation_restores_after_server_startup_error(request_cost_api_case): + state = request_cost_api_case("startup-error") + assert state["error"] is state["expected_error"] + assert state["calls"] == 0 and state["rows"][-1] == [3, 0, 0, 0, 0] + + +def test_native_api_records_all_nine_boundaries_only_for_existing_samples(request_cost_api_case): + state = request_cost_api_case("phases") + assert state["error"] is None + rows = state["rows"] + phases = [row for row in rows if row[0] == 1 and row[2] >= 10] + assert len(phases) == 54 + for ordinal, digest in state["tags"].items(): + selected = [row for row in phases if row[1] == ordinal] + assert [row[2] for row in selected] == list(range(10, 19)) + assert all(row[6] == digest and row[8:] == [0, 0, 0] for row in selected) + assert rows[-1] == [3, 18432, 72, 0, 0] + assert state["calls"] == 18432 and state["sends"] == 2 * 18432 From d4e2df6ec94c69996960bc205f654f1892437ca5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 12:52:34 +0000 Subject: [PATCH 3/7] Bound JSON encoder setup in Cell proof regression cases Exercise each default-field refusal and JSON numeric distinction before consolidating repeated default comparisons. Signed-off-by: zlareb1 (cherry picked from commit 73ef63b0b7b5e65254dc2618d19c267017fed904) Signed-off-by: zlareb1 --- .../test_cell_trigger_default_comparison.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/test_cell_trigger_default_comparison.py diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) From 7a39f99fa44d6e9daeccf0bbe95482c46a165a5c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 13:14:00 +0000 Subject: [PATCH 4/7] Compare Cell proof defaults in one canonical mapping Keep every default-field and JSON type check while avoiding a separate encoder setup for each field. Signed-off-by: zlareb1 (cherry picked from commit 0bfc712dcca9684e362af47a7e924432e87e53dd) Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/capture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", From 7c4ceb3711a2cdfcf4cd6af095ee1207ff56f72a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:09:53 +0000 Subject: [PATCH 5/7] Scope person-query observations to the decision task Shared client schema reads run with an independent context. Record only decision-tagged statements while retaining unbounded decision reads as failures. Signed-off-by: zlareb1 (cherry picked from commit 57327d5ea5778042f860cadf6cebec9eebb25821) Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 8 +- .../test_scim_person_query_observation.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_scim_person_query_observation.py diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75 From 9765f694765908c2b532a252a5edda59eb3e0ef2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:25:38 +0000 Subject: [PATCH 6/7] Cleanup tests measure their shared idle deadline Keep the blocked close, deadline observers and loop timers on one private clock. A controlled independent clock advance reproduces the elapsed-time dependency while the original cleanup and refusal assertions stay intact. Signed-off-by: zlareb1 (cherry picked from commit 4b03db4d8e5ac44d69af2146c371066907a5ec06) Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_cleanup.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) From feb5f0810bb18fe2ec3b4386cb9e4da2d112f618 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 23:41:29 +0000 Subject: [PATCH 7/7] Graph stats cancellation test uses a private idle clock Signed-off-by: zlareb1 (cherry picked from commit 4a183d834fd5ff315cb1b417700f03f7e2941253) Signed-off-by: zlareb1 --- .../unit/test_code_graph_stats_pass_budget.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/tests/unit/test_code_graph_stats_pass_budget.py b/tests/unit/test_code_graph_stats_pass_budget.py index a7e194e2f7..cd39781c4d 100644 --- a/tests/unit/test_code_graph_stats_pass_budget.py +++ b/tests/unit/test_code_graph_stats_pass_budget.py @@ -22,6 +22,7 @@ from typing import Any import pytest +import pytest_asyncio from server.code.api.models_status import CodeIndexStatusOutput from server.code.app import index_status @@ -607,6 +608,8 @@ async def test_a_cancelled_leg_issues_no_further_statements( assert repo.rows("code_unit_risk") == [] +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_pass_deadline_clock") async def test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -836,3 +839,101 @@ async def test_graph_stats_state_shows_unit_risk_progress() -> None: ) == "" ) + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.25]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_pass_deadline_clock") +async def test_pass_deadline_fixture_ignores_external_scheduler_elapsed(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + actual_pass = cgs.run_pass + actual_create_task = asyncio.create_task + owned = [] + + def monotonic(): + return real_monotonic() + elapsed + + async def run_pass(): + nonlocal elapsed + elapsed += scheduler_s + await asyncio.sleep(0.001) + return await actual_pass() + + def create_task(coro, *args, **kwargs): + task = actual_create_task(coro, *args, **kwargs) + owned.append(task) + return task + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cgs, "run_pass", run_pass) + monkeypatch.setattr(asyncio, "create_task", create_task) + try: + await test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count( + monkeypatch + ) + finally: + for task in owned: + if not task.done(): + task.cancel() + await asyncio.gather(*owned, return_exceptions=True) + + +@pytest_asyncio.fixture(loop_scope="function") +async def _pass_deadline_clock(monkeypatch): + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + async def to_thread(func, /, *args, **kwargs): + return func(*args, **kwargs) + + # Real pure calculations stay on this private clock instead of an executor. + monkeypatch.setattr(asyncio, "to_thread", to_thread) + # The cancellation protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cgs, "time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "graph-stats deadline leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_pass_deadline_clock_cancels_a_genuinely_blocked_query( + monkeypatch, _pass_deadline_clock +): + loop = asyncio.get_running_loop() + clock = _pass_deadline_clock + assert loop.time.__self__ is clock and cgs.time is clock + actual_edges = _Repo.edges_for_symbols + cancelled = [] + + async def edges(repo, *args, **kwargs): + started = loop.time() + try: + return await actual_edges(repo, *args, **kwargs) + except asyncio.CancelledError: + cancelled.append(loop.time() - started) + raise + + monkeypatch.setattr(_Repo, "edges_for_symbols", edges) + started = loop.time() + await test_a_pass_past_its_deadline_raises_timeout_and_freezes_the_statement_count(monkeypatch) + assert cancelled and all(elapsed == pytest.approx(0.2) for elapsed in cancelled) + assert loop.time() - started == pytest.approx(0.25)