Skip to content
Open
6 changes: 4 additions & 2 deletions scripts/gates/testsupport/cell_trigger/capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver):
"workload semantic/active",
)
need(row["confidence"] == 1.0, "workload confidence")
for field, default in self.p["sparse_defaults"].items():
equal(row[field], default)
equal(
{field: row[field] for field in self.p["sparse_defaults"]},
self.p["sparse_defaults"],
)
need(
row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key),
"exact stored payload/hash",
Expand Down
3 changes: 2 additions & 1 deletion server/memory/store/key_write_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from server.core.errors import PreconditionFailedError
from server.core.leases import LeaseBusy, LeaseLost
from server.core.repo_base import _CHClient, _default_client
from server.observability import degradations
from server.observability import degradations, provider_health


def lock_name(
Expand Down Expand Up @@ -109,6 +109,7 @@ async def _take(self, name: str, until: float) -> leases.Lease:
) from e
if lease is None:
raise _busy()
provider_health.record_success("memory_key_lock")
return lease

@asynccontextmanager
Expand Down
8 changes: 6 additions & 2 deletions tests/integration/test_scim_person_live_ch.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]:
query_all, query_one = client.query_all, client.query_one

async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any:
observing_person = ch.current_query_tag() == "auth.scim_people"
answer = await query_all(sql, params=params, **kwargs)
made.results.append((sql, ch.statement_settings(sql), answer))
if observing_person:
made.results.append((sql, ch.statement_settings(sql), answer))
return answer

async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any:
observing_person = ch.current_query_tag() == "auth.scim_people"
answer = await query_one(sql, params=params, **kwargs)
made.results.append((sql, ch.statement_settings(sql), answer))
if observing_person:
made.results.append((sql, ch.statement_settings(sql), answer))
return answer

monkeypatch.setattr(client, "query_all", all_rows)
Expand Down
72 changes: 71 additions & 1 deletion tests/unit/test_cell_attach_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from unittest.mock import AsyncMock

import pytest
import pytest_asyncio

from server.cli import cell_attach, cell_control
from tests.unit.cell_control_owner import owner
Expand Down Expand Up @@ -96,7 +97,8 @@ async def work():
await asyncio.gather(*owned_tasks, return_exceptions=True)


@pytest.mark.asyncio
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance():
closing = asyncio.Event()
conn = AsyncMock()
Expand Down Expand Up @@ -152,3 +154,71 @@ async def open(owned, what):
if failed:
assert caught.value is original
conn.close.assert_awaited_once()


@pytest.mark.parametrize("scheduler_s", [0.0, 0.16])
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s):
import time as wall_time

real_monotonic = wall_time.monotonic
elapsed = 0.0

def monotonic():
return real_monotonic() + elapsed

actual_close = cell_control.Owner.close

async def close(owned):
nonlocal elapsed
try:
return await actual_close(owned)
finally:
elapsed += scheduler_s

monkeypatch.setattr(wall_time, "monotonic", monotonic)
monkeypatch.setattr(cell_control.Owner, "close", close)
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()


@pytest_asyncio.fixture(loop_scope="function")
async def _cleanup_clock(monkeypatch):
from server.core import pg
from tests.unit.clock_fake import ManualClock

before = set(asyncio.all_tasks())
loop = asyncio.get_running_loop()
clock = ManualClock(t=loop.time())
selector = loop._selector
real_select = selector.select

def select(timeout=None):
if timeout is not None:
clock.advance(max(0.0, timeout))
timeout = 0
return real_select(timeout)

# The cleanup protocol spends idle time, independently of CPU scheduling.
monkeypatch.setattr(loop, "time", clock.monotonic)
monkeypatch.setattr(selector, "select", select)
monkeypatch.setattr(cell_control, "time", clock)
monkeypatch.setattr(pg, "time", clock)
monkeypatch.setattr(f"{__name__}.time", clock)
yield clock
leaked = asyncio.all_tasks() - before - {asyncio.current_task()}
for task in leaked:
task.cancel()
await asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "cell cleanup leaves no task on its private loop"


@pytest.mark.asyncio(loop_scope="function")
async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock):
from server.core import pg

loop = asyncio.get_running_loop()
assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic
began = _cleanup_clock.monotonic()
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()
assert _cleanup_clock.monotonic() - began == pytest.approx(0.03)
66 changes: 66 additions & 0 deletions tests/unit/test_cell_trigger_default_comparison.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Default-field proof checks keep their refusals without per-field JSON setup."""

from copy import deepcopy

import pytest

from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case
from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader
from tests.unit.test_cell_trigger_provenance_reuse import _case


def _expected(native, specification, context, row, audit, api):
native.Codec(specification).expected(
row,
audit,
context,
api=api,
metadata={"summary": {"jti": "b" * 32}},
driver={"host": "proof-host", "pid": 701},
)


@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"])
def test_default_proof_bounds_json_encoder_setup(
keeper_config_case, protocol_reader, monkeypatch, api
):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, api)
calls = 0
original = native.json.dumps

def counted(*args, **kwargs):
nonlocal calls
calls += 1
return original(*args, **kwargs)

monkeypatch.setattr(native.json, "dumps", counted)
_expected(native, specification, context, memories[0], audits[0], api)
# One row has 43 defaults; encoder setup must not repeat for each field.
assert calls <= 16


@pytest.mark.parametrize("index", range(43))
def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, False)
field, default = list(specification["sparse_defaults"].items())[index]
assert default != {"unexpected": True}
row = deepcopy(memories[0])
row[field] = {"unexpected": True}
with pytest.raises(native.Refused):
_expected(native, specification, context, row, audits[0], False)


@pytest.mark.parametrize("replacement", [False, 0.0])
def test_default_proof_keeps_json_numeric_type_refusals(
keeper_config_case, protocol_reader, replacement
):
native, *_ = keeper_config_case
specification, context, _, _, memories, audits, _ = _case(native, 1, False)
assert specification["sparse_defaults"]["pinned"] == 0
row = deepcopy(memories[0])
row["pinned"] = replacement
assert row["pinned"] == 0
with pytest.raises(native.Refused):
_expected(native, specification, context, row, audits[0], False)
138 changes: 138 additions & 0 deletions tests/unit/test_code_search_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
from __future__ import annotations

import asyncio
import sys
import time
from contextlib import contextmanager
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch

Expand Down Expand Up @@ -630,3 +633,138 @@ async def _put(*a: Any, **k: Any) -> None:
assert data["epistemic"] == "partial"
assert data["retrieval_work"]["vector_leg_failed"] == 1
assert puts == []


@contextmanager
def _first_content_clock():
from server.code.domain import deadline as code_deadline
from tests.unit.clock_fake import ManualClock

clock = ManualClock(t=1000.0)
real_asyncio = asyncio

def loop_factory():
loop = real_asyncio.new_event_loop()
selector = loop._selector
real_select = selector.select

def select(timeout=None):
events = real_select(0)
if not events and timeout is not None:
clock.advance(max(0.0, timeout))
return events

loop.time = clock.monotonic
selector.select = select
return loop

async def checked(coro):
before = real_asyncio.all_tasks()
try:
return await coro
finally:
leaked = real_asyncio.all_tasks() - before - {real_asyncio.current_task()}
for task in leaked:
task.cancel()
await real_asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "search deadline leaves no task on its private loop"

def run(coro):
with real_asyncio.Runner(loop_factory=loop_factory) as runner:
return runner.run(checked(coro))

# Fake leg sleeps and request budgets share idle time, independently of CPU scheduling.
timers = SimpleNamespace(
**{**vars(time), "monotonic": clock.monotonic, "perf_counter": clock.monotonic}
)
with (
patch.object(
sys.modules[__name__], "asyncio", SimpleNamespace(**{**vars(real_asyncio), "run": run})
),
patch.object(sys.modules[__name__], "time", timers),
patch.object(code_deadline, "time", timers),
patch.object(legs, "time", timers),
):
yield clock


@pytest.fixture(autouse=True)
def _first_content_deadline_clock(request):
if request.node.name != "test_the_deadline_waits_for_the_first_content_leg":
yield
return
with _first_content_clock():
yield


@pytest.mark.parametrize("jump", [0.0, 0.12])
def test_first_content_clock_ignores_unrelated_pre_leg_time(jump):
from server.code.domain import deadline as code_deadline

offset = 0.0
calls = []
real_monotonic = time.monotonic
actual_budget = cs_gather._leg_budget

def leg_budget(*args, **kwargs):
nonlocal offset
offset += jump
calls.append(offset)
return actual_budget(*args, **kwargs)

with (
patch.object(
code_deadline, "time", SimpleNamespace(monotonic=lambda: real_monotonic() + offset)
),
patch.object(cs_gather, "_leg_budget", leg_budget),
_first_content_clock(),
):
test_the_deadline_waits_for_the_first_content_leg()
assert calls == [jump]


def test_first_content_clock_still_drops_and_cancels_genuinely_blocked_legs():
cancelled = []

class BlockedRepo(_Repo):
async def lexical_search_chunks(self, *args, **kwargs):
try:
await asyncio.Event().wait()
finally:
cancelled.append("lexical")

async def vector_search_chunks(self, *args, **kwargs):
try:
await asyncio.Event().wait()
finally:
cancelled.append("vector")

with _first_content_clock() as clock:
_, res, wall = _run(BlockedRepo(), llm_legs="none", deadline_ms=400)
assert clock.t == pytest.approx(1000.4, abs=0.001)
assert 0.399 <= wall <= 0.401
assert res.units == [] and res.abstain and res.partial and res.epistemic == "partial"
assert set(res.legs_timed_out) == {"lexical", "vector"}
assert sorted(cancelled) == ["lexical", "vector"]


def test_first_content_clock_drains_and_rejects_an_orphan_task():
cancelled = []

async def orphan():
try:
await asyncio.Event().wait()
finally:
cancelled.append(True)

async def leak():
asyncio.create_task(orphan())
await asyncio.sleep(0)

real_asyncio, real_time = asyncio, time
real_run, real_monotonic = asyncio.run, time.monotonic
with _first_content_clock(), pytest.raises(AssertionError, match="leaves no task"):
asyncio.run(leak())
assert cancelled == [True]
assert asyncio is real_asyncio and asyncio.run is real_run
assert time is real_time and time.monotonic is real_monotonic
Loading