diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", diff --git a/server/memory/store/key_write_lock.py b/server/memory/store/key_write_lock.py index e0fb405404..bda6d2fdfd 100644 --- a/server/memory/store/key_write_lock.py +++ b/server/memory/store/key_write_lock.py @@ -20,7 +20,7 @@ from server.core.errors import PreconditionFailedError from server.core.leases import LeaseBusy, LeaseLost from server.core.repo_base import _CHClient, _default_client -from server.observability import degradations +from server.observability import degradations, provider_health def lock_name( @@ -109,6 +109,7 @@ async def _take(self, name: str, until: float) -> leases.Lease: ) from e if lease is None: raise _busy() + provider_health.record_success("memory_key_lock") return lease @asynccontextmanager diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) diff --git a/tests/unit/test_code_search_deadline.py b/tests/unit/test_code_search_deadline.py index 738495b27e..c7866d5243 100644 --- a/tests/unit/test_code_search_deadline.py +++ b/tests/unit/test_code_search_deadline.py @@ -15,7 +15,10 @@ from __future__ import annotations import asyncio +import sys import time +from contextlib import contextmanager +from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -630,3 +633,138 @@ async def _put(*a: Any, **k: Any) -> None: assert data["epistemic"] == "partial" assert data["retrieval_work"]["vector_leg_failed"] == 1 assert puts == [] + + +@contextmanager +def _first_content_clock(): + from server.code.domain import deadline as code_deadline + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=1000.0) + real_asyncio = asyncio + + def loop_factory(): + loop = real_asyncio.new_event_loop() + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + events = real_select(0) + if not events and timeout is not None: + clock.advance(max(0.0, timeout)) + return events + + loop.time = clock.monotonic + selector.select = select + return loop + + async def checked(coro): + before = real_asyncio.all_tasks() + try: + return await coro + finally: + leaked = real_asyncio.all_tasks() - before - {real_asyncio.current_task()} + for task in leaked: + task.cancel() + await real_asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "search deadline leaves no task on its private loop" + + def run(coro): + with real_asyncio.Runner(loop_factory=loop_factory) as runner: + return runner.run(checked(coro)) + + # Fake leg sleeps and request budgets share idle time, independently of CPU scheduling. + timers = SimpleNamespace( + **{**vars(time), "monotonic": clock.monotonic, "perf_counter": clock.monotonic} + ) + with ( + patch.object( + sys.modules[__name__], "asyncio", SimpleNamespace(**{**vars(real_asyncio), "run": run}) + ), + patch.object(sys.modules[__name__], "time", timers), + patch.object(code_deadline, "time", timers), + patch.object(legs, "time", timers), + ): + yield clock + + +@pytest.fixture(autouse=True) +def _first_content_deadline_clock(request): + if request.node.name != "test_the_deadline_waits_for_the_first_content_leg": + yield + return + with _first_content_clock(): + yield + + +@pytest.mark.parametrize("jump", [0.0, 0.12]) +def test_first_content_clock_ignores_unrelated_pre_leg_time(jump): + from server.code.domain import deadline as code_deadline + + offset = 0.0 + calls = [] + real_monotonic = time.monotonic + actual_budget = cs_gather._leg_budget + + def leg_budget(*args, **kwargs): + nonlocal offset + offset += jump + calls.append(offset) + return actual_budget(*args, **kwargs) + + with ( + patch.object( + code_deadline, "time", SimpleNamespace(monotonic=lambda: real_monotonic() + offset) + ), + patch.object(cs_gather, "_leg_budget", leg_budget), + _first_content_clock(), + ): + test_the_deadline_waits_for_the_first_content_leg() + assert calls == [jump] + + +def test_first_content_clock_still_drops_and_cancels_genuinely_blocked_legs(): + cancelled = [] + + class BlockedRepo(_Repo): + async def lexical_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("lexical") + + async def vector_search_chunks(self, *args, **kwargs): + try: + await asyncio.Event().wait() + finally: + cancelled.append("vector") + + with _first_content_clock() as clock: + _, res, wall = _run(BlockedRepo(), llm_legs="none", deadline_ms=400) + assert clock.t == pytest.approx(1000.4, abs=0.001) + assert 0.399 <= wall <= 0.401 + assert res.units == [] and res.abstain and res.partial and res.epistemic == "partial" + assert set(res.legs_timed_out) == {"lexical", "vector"} + assert sorted(cancelled) == ["lexical", "vector"] + + +def test_first_content_clock_drains_and_rejects_an_orphan_task(): + cancelled = [] + + async def orphan(): + try: + await asyncio.Event().wait() + finally: + cancelled.append(True) + + async def leak(): + asyncio.create_task(orphan()) + await asyncio.sleep(0) + + real_asyncio, real_time = asyncio, time + real_run, real_monotonic = asyncio.run, time.monotonic + with _first_content_clock(), pytest.raises(AssertionError, match="leaves no task"): + asyncio.run(leak()) + assert cancelled == [True] + assert asyncio is real_asyncio and asyncio.run is real_run + assert time is real_time and time.monotonic is real_monotonic diff --git a/tests/unit/test_memory_key_write_lock.py b/tests/unit/test_memory_key_write_lock.py index d09592ea7b..e491b6aeaf 100644 --- a/tests/unit/test_memory_key_write_lock.py +++ b/tests/unit/test_memory_key_write_lock.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from contextlib import nullcontext from types import SimpleNamespace from typing import Any @@ -13,6 +14,7 @@ from server.memory.app import update as update_mod from server.memory.app import write as write_mod from server.memory.store import key_write_lock as kl +from server.observability import provider_health from tests.unit.claims_fake import FakeLockTable _IDENT = ("org", "ns", "agent", "-", "-", "proc/deploy/steps") @@ -92,6 +94,46 @@ async def test_keeper_failure_refuses_the_write( assert notes == [("memory_key_lock", "keeper_unavailable")] +@pytest.mark.parametrize("body_fails", [False, True]) +async def test_key_lock_health_recovers_when_the_lease_is_acquired( + monkeypatch: pytest.MonkeyPatch, body_fails: bool +) -> None: + monkeypatch.setattr(provider_health, "_state", {}) + ch = FakeKeeper(fail=True) + locks = _locks(ch, monkeypatch) + with pytest.raises(BackendUnavailableError): + async with locks.hold(*_IDENT): + pytest.fail("entered without a lease") + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is True + + provider_health.record_failure("other_provider", "still unavailable") + ch.down = False + with pytest.raises(ValueError, match="body failed") if body_fails else nullcontext(): + async with locks.hold(*_IDENT): + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is False + if body_fails: + raise ValueError("body failed") + assert provider_health.status()["providers"]["other_provider"]["failing"] is True + assert ch.keys == {} and len(locks) == 0 + + +async def test_a_busy_key_does_not_clear_the_last_acquisition_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(provider_health, "_state", {}) + ch = FakeKeeper(fail=True) + locks = _locks(ch, monkeypatch) + with pytest.raises(BackendUnavailableError): + async with locks.hold(*_IDENT): + pytest.fail("entered without a lease") + ch.down = False + ch.seed(kl.lock_name(*_IDENT), "other") + with pytest.raises(PreconditionFailedError): + async with locks.hold(*_IDENT): + pytest.fail("entered a held lock") + assert provider_health.status()["providers"]["memory_key_lock"]["failing"] is True + + async def test_racing_conditional_sets_let_one_through( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_native_command_retirement.py b/tests/unit/test_native_command_retirement.py index 90cd15a024..f09c4ff187 100644 --- a/tests/unit/test_native_command_retirement.py +++ b/tests/unit/test_native_command_retirement.py @@ -212,10 +212,96 @@ async def refuse(_): assert not gate.enrolled and not gate.closed and events == [] +def _claim_async_state(gate, events, stdin_fd=None): + """Fixed metadata only: never coroutine arguments, locals, task names or exception text.""" + import asyncio + import os + import signal + from itertools import islice + from pathlib import Path + + handler = signal.getsignal(signal.SIGINT) + state = { + "sigint_blocked": signal.SIGINT in signal.pthread_sigmask(signal.SIG_BLOCK, []), + "sigint_pending": signal.SIGINT in signal.sigpending(), + "stdin_blocking": os.get_blocking(stdin_fd) if stdin_fd is not None else None, + "sigint_handler": ( + "default" + if handler is signal.default_int_handler + else "ignored" + if handler == signal.SIG_IGN + else "custom" + ), + "gate": { + key: getattr(gate, key) for key in ("active", "closed", "enrolled", "producers_stopped") + }, + "events": { + key: events.count(key) + for key in ( + "verified", + "enrolled", + "acceptance", + "lookup", + "release", + "buffer", + "certificate", + ) + }, + "tasks": [], + } + try: + loop = asyncio.get_running_loop() + except RuntimeError: + state["loop_running"] = False + return state + state["loop_running"] = True + tasks = asyncio.all_tasks(loop) + state["tasks_truncated"] = len(tasks) > 8 + for task in islice(tasks, 8): + frames = [] + coroutine = task.get_coro() + for _ in range(8): + frame = getattr(coroutine, "cr_frame", None) or getattr(coroutine, "gi_frame", None) + if frame is not None: + frames.append( + [ + Path(frame.f_code.co_filename).name[:64], + frame.f_code.co_name[:64], + frame.f_lineno, + ] + ) + coroutine = getattr(coroutine, "cr_await", None) or getattr( + coroutine, "gi_yieldfrom", None + ) + if coroutine is None: + break + state["tasks"].append({"cancelling": task.cancelling(), "frames": frames}) + return state + + +def _write_claim_async_state(trace_fd, gate, events, stdin_fd=None): + import contextlib + import json + import os + + try: + packet = json.dumps( + _claim_async_state(gate, events, stdin_fd), separators=(",", ":") + ).encode() + if len(packet) > 32700: + packet = b'{"observer_truncated":true}' + os.write(trace_fd, b"claim_async_state=" + packet + b"\n") + except Exception: # noqa: BLE001 — diagnostics must preserve the failed exit check. + with contextlib.suppress(OSError): + os.write(trace_fd, b'claim_async_state={"observer_failed":true}\n') + + def _claim_prompt_subprocess(result_path): """Run the real command and asyncio.run with only its external work faked.""" + import faulthandler import json import os + import signal import sys from pathlib import Path @@ -239,8 +325,21 @@ async def release(**_): monkeypatch.setattr(namespaces, "fetch_row", lookup) monkeypatch.setattr(claim_release, "release_claim", release) blocking = os.get_blocking(sys.stdin.fileno()) + trace_path = Path(result_path).with_suffix(".trace") + # The owned child keeps this descriptor open through interpreter shutdown. + trace_fd = os.open(trace_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.write( + trace_fd, + f"sigint_ignored={signal.getsignal(signal.SIGINT) == signal.SIG_IGN}\n".encode(), + ) outcome = None try: + + def diagnostic(_signum, _frame): + _write_claim_async_state(trace_fd, gate, events, sys.stdin.fileno()) + + signal.signal(signal.SIGUSR1, diagnostic) + faulthandler.register(signal.SIGUSR1, file=trace_fd, all_threads=True, chain=True) outcome = access._cmd_claim_release(SimpleNamespace(org="o", namespace="n", yes=False)) except KeyboardInterrupt: outcome = "interrupted" @@ -269,9 +368,16 @@ def claim_prompt_child(tmp_path): owned = [] - def start(mode="pipe", contents=b""): + def start(mode="pipe", contents=b"", *, stalled=False): result = tmp_path / f"claim-{len(owned)}.json" - child = SimpleNamespace(process=None, writer=None, reader=None, file=None, result=result) + child = SimpleNamespace( + process=None, + writer=None, + reader=None, + file=None, + result=result, + trace=result.with_suffix(".trace"), + ) owned.append(child) if mode == "regular": child.file = tempfile.TemporaryFile() @@ -289,8 +395,9 @@ def start(mode="pipe", contents=b""): sys.executable, "-B", "-c", - "from tests.unit.test_native_command_retirement import _claim_prompt_subprocess; " - "import sys; _claim_prompt_subprocess(sys.argv[1])", + "from tests.unit.test_native_command_retirement import " + + ("_claim_prompt_stalled_subprocess" if stalled else "_claim_prompt_subprocess") + + " as run; import sys; run(sys.argv[1])", str(result), ], cwd=Path(__file__).resolve().parents[2], @@ -366,7 +473,19 @@ def test_actual_claim_prompt_cancellation_exits_with_stdin_still_open( try: child.process.wait(timeout=5) except subprocess.TimeoutExpired: - pytest.fail("cancelled claim-release did not exit while stdin remained open") + # The exit deadline has already failed; collect only our child's bounded evidence. + diagnostic = b"child diagnostic unavailable" + try: + child.process.send_signal(signal.SIGUSR1) + time.sleep(0.5) + with child.trace.open("rb") as trace: + diagnostic = trace.read(65536) + except OSError as error: + diagnostic = type(error).__name__.encode() + pytest.fail( + "cancelled claim-release did not exit while stdin remained open\n" + + diagnostic.decode(errors="replace") + ) assert not child.writer.closed # communicate() or EOF must not make the assertion pass. assert child.process.returncode == 0 result = json.loads(child.result.read_text()) @@ -411,3 +530,118 @@ def test_actual_claim_prompt_answer_and_eof_controls( assert result["events"].index("lookup") < result["events"].index("certificate") assert result["events"].count("acceptance") == (2 if expected == 0 else 1) assert result["events"].count("release") == (1 if expected == 0 else 0) + + +def _claim_prompt_stalled_subprocess(result_path): + """Signal the actual command before publishing the observer's ready marker.""" + import asyncio + import os + import signal + from pathlib import Path + + async def stalled_answer(): + os.kill(os.getpid(), signal.SIGINT) + try: + await asyncio.Future() + except asyncio.CancelledError: + Path(result_path).with_suffix(".stalled").touch() + print("Continue? [y/N] ", end="", flush=True) + await asyncio.Future() + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(access, "_claim_release_answer", stalled_answer) + _claim_prompt_subprocess(result_path) + + +def test_claim_cancel_diagnostic_exposes_suspended_native_coroutine(claim_prompt_child): + import json + import signal + import time + + child = claim_prompt_child(stalled=True) + _wait_for_claim_prompt(child) + assert child.result.with_suffix(".stalled").exists() + child.process.send_signal(signal.SIGUSR1) + end = time.monotonic() + 2 + trace = "" + while "claim_async_state=" not in trace: + assert time.monotonic() < end, "the trace omitted the suspended native command" + trace = child.trace.read_bytes()[:65536].decode(errors="replace") + time.sleep(0.01) + packet = json.loads(trace.split("claim_async_state=", 1)[1].splitlines()[0]) + frames = [frame for task in packet["tasks"] for frame in task["frames"]] + assert any(frame[1] == "stalled_answer" for frame in frames) + assert any(task["cancelling"] == 1 for task in packet["tasks"]) + assert packet["gate"] == { + "active": 1, + "closed": False, + "enrolled": True, + "producers_stopped": False, + } + assert not packet["sigint_blocked"] and not packet["sigint_pending"] + assert "Current thread" in trace + child.process.send_signal(signal.SIGINT) + child.process.wait(timeout=5) + assert child.process.returncode == 0 + result = json.loads(child.result.read_text()) + assert result["outcome"] == "interrupted" and result["quiescent"] + + +def test_claim_cancel_diagnostic_limits_tasks_frames_and_metadata(): + import asyncio + import json + + async def run(): + async def nested(depth): + if depth: + await nested(depth - 1) + else: + await asyncio.Future() + + tasks = [asyncio.create_task(nested(20), name="private-task-name") for _ in range(20)] + try: + await asyncio.sleep(0) + state = _claim_async_state(retirement.RetirementGate(), ["private-event"]) + assert state["loop_running"] and state["tasks_truncated"] + assert len(state["tasks"]) == 8 + assert all(len(task["frames"]) <= 8 for task in state["tasks"]) + packet = json.dumps(state) + assert len(packet.encode()) < 32768 + assert "private-task-name" not in packet and "private-event" not in packet + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + asyncio.run(run()) + + +def test_claim_cancel_diagnostic_without_running_loop(): + state = _claim_async_state(retirement.RetirementGate(), []) + assert not state["loop_running"] and state["tasks"] == [] + + +def test_claim_cancel_diagnostic_failure_has_no_exception_text(monkeypatch): + import os + import sys + + def failed(*_): + raise ValueError("private exception detail") + + packets = [] + monkeypatch.setattr(sys.modules[__name__], "_claim_async_state", failed) + monkeypatch.setattr(os, "write", lambda fd, packet: packets.append((fd, packet))) + _write_claim_async_state(42, retirement.RetirementGate(), []) + assert packets == [(42, b'claim_async_state={"observer_failed":true}\n')] + + monkeypatch.setattr( + sys.modules[__name__], "_claim_async_state", lambda *_: {"large": "x" * 40000} + ) + _write_claim_async_state(42, retirement.RetirementGate(), []) + assert packets[-1] == (42, b'claim_async_state={"observer_truncated":true}\n') + + def closed(*_): + raise OSError("private descriptor detail") + + monkeypatch.setattr(os, "write", closed) + _write_claim_async_state(42, retirement.RetirementGate(), []) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75