From e3cdd7357b93f7d72857c6e6c97aec27b6b95085 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 21:25:08 +0000 Subject: [PATCH 1/5] Recheck renewed supervisor leases after a wait expires A confirmed renewal can extend the deadline while the previous wait resumes. Use the current freshness before stopping workers, and retain genuine expiry controls. Signed-off-by: zlareb1 --- server/workers/leader.py | 3 ++ tests/unit/test_supervisor_lease.py | 48 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/server/workers/leader.py b/server/workers/leader.py index c5c697dfa0..10f7ceecd2 100644 --- a/server/workers/leader.py +++ b/server/workers/leader.py @@ -334,6 +334,9 @@ async def _until_done( done, _ = await asyncio.wait({fut}, timeout=timeout) if done: return running + # A renewal can land while the previous deadline's wait resumes. + if lease.fresh(): + continue log.warning( "supervisor lease: no renewal within %.0f s; stopping the workers", float(settings.supervisor_lease_ttl_s) / 2, diff --git a/tests/unit/test_supervisor_lease.py b/tests/unit/test_supervisor_lease.py index 298b87d933..a13a1b0d2f 100644 --- a/tests/unit/test_supervisor_lease.py +++ b/tests/unit/test_supervisor_lease.py @@ -345,6 +345,54 @@ def test_a_holder_that_gave_up_takes_its_own_key_back_on_the_next_round() -> Non assert "holds workers_supervisor" in lease.reason +@pytest.mark.parametrize( + ("renew_at", "resume_at", "should_run"), + [(10.0, 16.0, True), (14.0, 20.0, True), (None, 16.0, False), (10.0, 26.0, False)], +) +async def test_old_wait_timeout_rechecks_confirmed_renewal( + monkeypatch: pytest.MonkeyPatch, + renew_at: float | None, + resume_at: float, + should_run: bool, +) -> None: + now = [0.0] + lock = _Lock([True], cfg=leader.lease_config(30)) + lock.clock = lambda: now[0] + monkeypatch.setattr(leader, "_stand_aside_reason", lambda: "") + monkeypatch.setattr(leader, "newer_build", lambda _name: 0) + lease = _auto(lock) + lease.step() + pending = asyncio.get_running_loop().create_future() + stops: list[bool] = [] + waits: list[float | None] = [] + + async def on_lost() -> None: + stops.append(True) + + async def stale_wait( + fs: set[Any], + *, + timeout: float | None = None, # noqa: ASYNC109 — mirrors asyncio.wait's keyword + ) -> Any: + assert fs == {pending} + waits.append(timeout) + if len(waits) == 1: + assert timeout == 15.0 + if renew_at is not None: + now[0] = renew_at + lock.renew_if_due() + now[0] = resume_at + return set(), {pending} + pending.set_result(None) + return {pending}, set() + + monkeypatch.setattr(leader.asyncio, "wait", stale_wait) + running = await leader._until_done(pending, lease, True, on_lost) + assert running is should_run + assert stops == ([] if should_run else [True]) + assert waits == [15.0, (renew_at + 15.0 - resume_at) if should_run else None] + + async def test_the_workers_stop_at_the_deadline_while_a_renewal_blocks( monkeypatch: pytest.MonkeyPatch, ) -> None: From 84a0d8d9440ad3e62345a5679d410327263a0a3e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 12:52:34 +0000 Subject: [PATCH 2/5] Bound JSON encoder setup in Cell proof regression cases Exercise each default-field refusal and JSON numeric distinction before consolidating repeated default comparisons. Signed-off-by: zlareb1 (cherry picked from commit 73ef63b0b7b5e65254dc2618d19c267017fed904) Signed-off-by: zlareb1 --- .../test_cell_trigger_default_comparison.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/test_cell_trigger_default_comparison.py diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) From 4f81ed082190eb0dbe2ca4dd709796095c840807 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 13:14:00 +0000 Subject: [PATCH 3/5] Compare Cell proof defaults in one canonical mapping Keep every default-field and JSON type check while avoiding a separate encoder setup for each field. Signed-off-by: zlareb1 (cherry picked from commit 0bfc712dcca9684e362af47a7e924432e87e53dd) Signed-off-by: zlareb1 --- scripts/gates/testsupport/cell_trigger/capture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", From a9cc3c6caea52b66f39776d508a6355bbb3bce2b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:09:53 +0000 Subject: [PATCH 4/5] Scope person-query observations to the decision task Shared client schema reads run with an independent context. Record only decision-tagged statements while retaining unbounded decision reads as failures. Signed-off-by: zlareb1 (cherry picked from commit 57327d5ea5778042f860cadf6cebec9eebb25821) Signed-off-by: zlareb1 --- tests/integration/test_scim_person_live_ch.py | 8 +- .../test_scim_person_query_observation.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_scim_person_query_observation.py diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75 From 2bd301c28649b229ffb8410b57c744f1b5f4ff0f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:25:38 +0000 Subject: [PATCH 5/5] Cleanup tests measure their shared idle deadline Keep the blocked close, deadline observers and loop timers on one private clock. A controlled independent clock advance reproduces the elapsed-time dependency while the original cleanup and refusal assertions stay intact. Signed-off-by: zlareb1 (cherry picked from commit 4b03db4d8e5ac44d69af2146c371066907a5ec06) Signed-off-by: zlareb1 --- tests/unit/test_cell_attach_cleanup.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03)