From ae10ca639d36f76564014a573252b3e8d286ea38 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 14:16:18 +0000 Subject: [PATCH 1/3] Restore fixtures spend simulated idle time on key locks Signed-off-by: zlareb1 --- tests/unit/test_restore_unsupersede.py | 42 +++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_restore_unsupersede.py b/tests/unit/test_restore_unsupersede.py index 6bdab5d788..17192d5f7f 100644 --- a/tests/unit/test_restore_unsupersede.py +++ b/tests/unit/test_restore_unsupersede.py @@ -4,23 +4,51 @@ from __future__ import annotations +import asyncio +from collections.abc import AsyncIterator from datetime import UTC, datetime, timedelta from typing import Any import pytest +import pytest_asyncio +from server.core import leases from server.memory import store as memory_repo -from server.memory.store import MemoryRow +from server.memory.store import MemoryRow, key_write_lock from tests.unit.claims_fake import fake_key_locks +from tests.unit.clock_fake import ManualClock from tests.unit.memory_store import stub_status_before pytestmark = pytest.mark.usefixtures("no_blacklisted_values") -@pytest.fixture(autouse=True) -def _key_locks(monkeypatch: pytest.MonkeyPatch) -> None: - # The Undo holds the key's write lock, a Keeper lease: here over a fake lease table. +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def _key_locks(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[None]: + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # An uncontended fake lease spends idle time, not CPU scheduling time. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(key_write_lock, "time", clock) + monkeypatch.setattr(leases, "time", clock) fake_key_locks(monkeypatch) + monkeypatch.setattr(key_write_lock.key_write_locks.ch, "now", clock.monotonic) + yield + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "restoration leaves no task on its private loop" @pytest.fixture(autouse=True) @@ -68,7 +96,7 @@ def _superseded_row() -> MemoryRow: ) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") async def test_restore_reverses_an_automatic_supersession_and_writes_the_ledger_row( monkeypatch, earlier_status_reads ): @@ -154,7 +182,7 @@ async def _unsupersede(*a: Any, **k: Any) -> int: assert subj["version"] == out.version, "the resolver reads the restore's version" -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") async def test_a_retried_undo_that_landed_late_answers_restored_when_its_search_document_fails( monkeypatch, caplog ): @@ -192,7 +220,7 @@ async def _reopen(_input: Any, _existing: Any, version: int, **_: Any) -> None: assert "search document failed for k" in caplog.text -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") async def test_an_undo_whose_record_fails_restores_nothing(monkeypatch): # The resolver honours an Undo only by its record: an Undo it could not see would # be judged again, so a failed record fails the Undo before the restore is written. From 3a86620813be2baf1df291b89892125f9e4f09f2 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 14:21:46 +0000 Subject: [PATCH 2/3] Restore tests share the fixture loop and reject blocked locks Signed-off-by: zlareb1 --- tests/unit/test_restore_unsupersede.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/test_restore_unsupersede.py b/tests/unit/test_restore_unsupersede.py index 17192d5f7f..9b2317f85b 100644 --- a/tests/unit/test_restore_unsupersede.py +++ b/tests/unit/test_restore_unsupersede.py @@ -248,6 +248,7 @@ async def _boom(*_: Any, **__: Any) -> str: @pytest.mark.parametrize("deleted_since", [True, False]) +@pytest.mark.asyncio(loop_scope="function") async def test_an_undo_after_a_later_delete_leaves_the_delete(monkeypatch, deleted_since): # The first read still shows the retirement; the read-your-writes head shows the # delete that landed since, or no row at all. The delete stands, and the answer @@ -282,6 +283,7 @@ async def _get_by_key(*_: Any, **__: Any) -> MemoryRow | None: assert out.memory_id is None +@pytest.mark.asyncio(loop_scope="function") async def test_a_guarded_restore_never_undoes_a_delete(monkeypatch): # The inspector showed the row retired; another client deleted it since. from server.memory.api.models_crud import MemoryRestoreInput @@ -337,6 +339,7 @@ async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: ({"status": "active", "valid_to": _PAST}, "active"), ], ) +@pytest.mark.asyncio(loop_scope="function") async def test_undoing_a_retired_duplicate_keeps_it_a_duplicate(monkeypatch, keeper, status): # The same-key judge can retire a duplicate row. Undone while the row it # duplicates is still served, it stays hidden behind that row, on both tables. @@ -389,3 +392,20 @@ def test_a_version_becomes_its_time_to_the_microsecond() -> None: # A float of these ns rounds the microsecond up; integer arithmetic does not. assert version_time(1_792_383_601_496_448_999) == datetime(2026, 10, 19, 4, 20, 1, 496448) + + +@pytest.mark.asyncio(loop_scope="function") +async def test_restore_clock_owns_the_test_loop(): + from server.core.errors import PreconditionFailedError + + loop = asyncio.get_running_loop() + clock = key_write_lock.time + assert loop.time == clock.monotonic + began = clock.monotonic() + locks = key_write_lock.key_write_locks + async with locks.hold("o", "n", "a", "-", "s", "clock-control"): + with pytest.raises(PreconditionFailedError): + async with locks.hold("o", "n", "a", "-", "s", "clock-control"): + pytest.fail("a blocked key lock cannot outlive its idle budget") + assert clock.monotonic() > began + assert len(locks) == 0 From 8c4758c645f14362507d2273758b375e01861ab9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:09:53 +0000 Subject: [PATCH 3/3] Scope person-query observations to the decision task Shared client schema reads run with an independent context. Record only decision-tagged statements while retaining unbounded decision reads as failures. Signed-off-by: zlareb1 (cherry picked from commit 57327d5ea5778042f860cadf6cebec9eebb25821) Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 8 +- .../test_scim_person_query_observation.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_scim_person_query_observation.py diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75