diff --git a/docs/architecture/memory-architecture.md b/docs/architecture/memory-architecture.md index aadff6b6bc..da32f0d8c0 100644 --- a/docs/architecture/memory-architecture.md +++ b/docs/architecture/memory-architecture.md @@ -1001,6 +1001,10 @@ Session digests carry the keys they compressed in `derived_from`. Before dedupli the summary leg checks those known sources under the digest's full identity. A source that is missing, unserved, deleted, expired, outside its validity, high-sensitivity or at a newer version withholds the digest. An unreadable source check withholds all unchecked digests. +Under Postgres authority, the leg reads each exact source key again after reading its +projection. The authoritative version, content hash and serving metadata must match +the projection; a missing head or content hash withholds the digest. This check reads +metadata only, and never falls back to the projection when Postgres is unavailable. Without source revision hashes, a later source version conservatively withholds a digest during replay too. Lineage covers extracted facts, not every sentence of the transcript: a digest without a known source remains session evidence, and does not prove that its diff --git a/scripts/gates/testsupport/cell_trigger/capture.py b/scripts/gates/testsupport/cell_trigger/capture.py index 635df62581..0360fbdb8c 100644 --- a/scripts/gates/testsupport/cell_trigger/capture.py +++ b/scripts/gates/testsupport/cell_trigger/capture.py @@ -255,8 +255,10 @@ def expected(self, row, audit, context, *, api, metadata, driver): "workload semantic/active", ) need(row["confidence"] == 1.0, "workload confidence") - for field, default in self.p["sparse_defaults"].items(): - equal(row[field], default) + equal( + {field: row[field] for field in self.p["sparse_defaults"]}, + self.p["sparse_defaults"], + ) need( row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key), "exact stored payload/hash", diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index ef45d63441..6420e06925 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -478,6 +478,37 @@ def _key_arrays(keys: Sequence[IdentityKey]) -> list[list[str]]: _HEADS_BY_KEYS = f"SELECT c.row {_EXACT_KEYS}" +# A digest may use the projection's eligibility decision only while these fields match +# its authority. Read no source values: a shortlist can name many source keys. +SUMMARY_SOURCE_STATE_COLUMNS = ( + "version", + "content_sha256", + "status", + "sensitivity", + "tombstoned_at", + "redacted_at", + "expires_at", + "valid_to", + "created_at", + "valid_at", + "temporal_anchor", + "invalid_at", +) +_SUMMARY_SOURCE_COLUMNS = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + *SUMMARY_SOURCE_STATE_COLUMNS, +) +_SUMMARY_SOURCE_HEADS = ( + "SELECT jsonb_strip_nulls(jsonb_build_object(" + + ", ".join(f"'{col}', c.row->'{col}'" for col in _SUMMARY_SOURCE_COLUMNS) + + f")) AS row {_EXACT_KEYS}" +) + _HEADS_OWING = f""" SELECT c.row, coalesce(d.targets, 0) AS owed {_EXACT_KEYS} LEFT JOIN loom.memory_deliveries d @@ -1607,6 +1638,27 @@ async def heads_by_keys( return {identity_key(r["row"]): decode_row(r["row"]) for r in rows} +async def summary_source_heads( + keys: Sequence[IdentityKey], +) -> dict[IdentityKey, dict[str, Any]]: + """Exact-key digest source metadata, including deleted and unserved heads. + + A missing or unreadable head cannot validate a projected source. Values and other + payloads stay in the authority; the caller compares only the serving state. + """ + keys = [k for k in keys if not pg.unstorable(k)] + if not keys: + return {} + rows = await _read(_SUMMARY_SOURCE_HEADS, *_key_arrays(keys), durable=False) + heads = {} + for row in rows: + decoded = decode_row(row["row"]) + heads[identity_key(decoded)] = { + col: decoded.get(col) for col in SUMMARY_SOURCE_STATE_COLUMNS + } + return heads + + async def heads_owing( keys: Sequence[IdentityKey], *, durable: bool = False ) -> dict[IdentityKey, tuple[dict[str, Any], int]]: diff --git a/server/memory/store/summary_sources.py b/server/memory/store/summary_sources.py index 2be31f0c33..22a931abd7 100644 --- a/server/memory/store/summary_sources.py +++ b/server/memory/store/summary_sources.py @@ -2,14 +2,20 @@ from __future__ import annotations +from datetime import datetime from typing import Any +from server.core import pg +from server.core.datetimes import to_naive_utc +from server.core.hashing import hash_text from server.core.repo_base import _CHClient, query_all_sliced +from server.memory.domain.redaction_lineage import is_content_hash from server.memory.domain.temporal_sql import ( as_of_predicate, status_predicate, tombstone_predicate, ) +from server.memory.store import pg_memory SourceKey = tuple[str, str, str, str] @@ -21,6 +27,18 @@ def _sources(row: dict[str, Any]) -> list[SourceKey]: return [(agent, user, session, str(key)) for key in row.get("derived_from", []) if key] +def _source_state(row: dict[str, Any]) -> tuple[Any, ...]: + return tuple( + hash_text(value) + if col == "content_sha256" + else to_naive_utc(value) + if isinstance(value, datetime) + else value + for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS + for value in [row.get(col)] + ) + + async def current_summary_sources( ch: _CHClient, rows: list[dict[str, Any]], @@ -43,6 +61,7 @@ async def current_summary_sources( tombstone_predicate(as_of_basis, replaying=replaying), status_predicate(as_of_basis, replaying=replaying), "sensitivity != 'high'", + "redacted_at IS NULL", "(expires_at IS NULL OR expires_at > now())", "(valid_to IS NULL OR valid_to > now())", ] @@ -55,7 +74,8 @@ async def current_summary_sources( f""" SELECT agent, if(user_id = '', '-', user_id) AS source_user, if(session_id = '', '-', session_id) AS source_session, memory_key, - version, {" AND ".join(state)} AS source_served + {", ".join(pg_memory.SUMMARY_SOURCE_STATE_COLUMNS)}, + {" AND ".join(state)} AS source_served FROM loom.memory FINAL WHERE org = {{org:String}} AND namespace = {{ns:String}} AND (agent, if(user_id = '', '-', user_id), if(session_id = '', '-', session_id), @@ -64,6 +84,11 @@ async def current_summary_sources( params, "sources", ) + authority = ( + await pg_memory.summary_source_heads([(org, namespace, *key) for key in sources]) + if pg.enabled() + else None + ) heads: dict[SourceKey, tuple[bool, int]] = {} for p in parents: key = ( @@ -73,7 +98,18 @@ async def current_summary_sources( str(p["memory_key"]), ) served, version = heads.get(key, (True, 0)) - heads[key] = (served and bool(p["source_served"]), max(version, int(p["version"]))) + current = True + if authority is not None: + head = authority.get((org, namespace, *key)) + current = ( + head is not None + and is_content_hash(str(head.get("content_sha256") or "")) + and _source_state(head) == _source_state(p) + ) + heads[key] = ( + served and bool(p["source_served"]) and current, + max(version, int(p["version"])), + ) return [ row for row in rows diff --git a/tests/analyzer/cases/legs.py b/tests/analyzer/cases/legs.py index e0e78e805b..8bb5a7ac8a 100644 --- a/tests/analyzer/cases/legs.py +++ b/tests/analyzer/cases/legs.py @@ -713,6 +713,21 @@ def cases() -> list[Case]: "as_of": (False, True), "basis": ("valid_time", "knowledge_time"), }, + rows=rows_by_substring( + { + "FROM loom.session_summaries FINAL": [ + { + "agent": AGENT, + "user_id": "alice", + "session_id": "session", + "version": 1, + "derived_from": ["source"], + "summary_text": "a source fact", + "distance": 0.1, + } + ] + } + ), ), Case( "memory_repo.reads", diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 955a96a71a..ed424c2d2e 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -991,6 +991,7 @@ 764e0e259701 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 765404bd5d8d SELECT toDate(coalesce(temporal_anchor, valid_at)) AS event_date, cosineDistance(embedding, {q:Array(Float32)}) AS distance, substring(content_excerpt, ?, ?) AS excerpt, memory_key, temporal_anchor IS 7662b4971175 SELECT org, namespace, agent, user_id, session_id, memory_id, memory_key, memory_type, value, confidence, source_trace, version, tombstoned_at, created_at, expires_at, sensitivity FROM loom.memory FIN +769c165248dc SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp 76ed76b066ae SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 76f47ab9df72 SELECT ? AS present FROM loom.code_caller_guards WHERE org = {org:String} AND namespace = {ns:String} LIMIT ? 77424b91d322 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} @@ -1649,6 +1650,7 @@ bd4266cbfe15 SELECT count() AS n FROM loom.conflict_allowlist FINAL WHERE org = bda43b1f632f WITH arrayConcat(splitByNonAlpha(context_lc), splitByNonAlpha(content_lc)) AS words SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, cosineDistance bde7c432bca1 ALTER TABLE loom.entity_communities DELETE WHERE (org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)})) AND ({ bde98d358a2b SELECT path, uniqExact(facet_kind, name) AS matched, groupArray((facet_kind, name, ordinal, line, confidence)) AS hits, length(arrayIntersect(groupUniqArrayIf(ordinal, facet_kind = {k0:String} AND nam +be08a1021716 SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp be29d7515a0a SELECT count() AS n FROM loom.memory_doc_stats WHERE org = {org:String} AND namespace = {ns:String} be6f152846ae SELECT arrayFirst( u -> startsWith(source_resource_id, concat({prefix:String}, u, ?)), {uris:Array(String)} ) AS source_uri, toUInt8(has(tags, {proposed_tag:String})) AS proposed, count() AS total, co be735bd4b1cf ALTER TABLE loom.namespace_guides DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -1932,6 +1934,7 @@ dbd04ba47fb3 SELECT agent, user_id, session_id, memory_key, memory_type, content dbe0f33a89a7 SELECT toString(memory_id) AS memory_id, org, namespace, agent, user_id, session_id, memory_key, value, created_at, valid_at, version, redacted_at, toString(content_sha256) AS content_sha256 FROM loom dbe745784521 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { dbefbb4f200e SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user +dbf0d4bbb01d SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp dbf737bf992e ALTER TABLE loom.knowledge_uploads DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? dbfe0a193e5b SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND superseded_at IS NULL dc003d9f3eb2 SELECT toString(failure_id) AS failure_id, org, namespace, agent, projection, source_operation, source_resource_id, memory_keys, error_type, error_message, payload, status, created_at, resolved_at FRO diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 10bd835305..3ff0b402d8 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -235,6 +235,37 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) +async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: + """Read only fixed status fields; a retry message can contain source content.""" + from server.knowledge.store import ingestions + + try: + async with asyncio.timeout(1): + row = await ingestions.get_ingestion(org, ns, ingestion_id) + except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion + return {"observation": "unavailable"} + if row is None: + return {"observation": "missing"} + choices = { + "state": {"pending", "processing", "ready", "partial", "failed", "cancelled"}, + "stage": {"", "parse", "embed", "publish"}, + "error_code": {"", "retrying", "retries_exhausted", "revision_conflict"}, + } + out: dict[str, Any] = {} + for key, allowed in choices.items(): + value = row.get(key) + out[key] = ( + value + if type(value) is str and value in allowed + else "other" + if key == "error_code" + else "unknown" + ) + attempts = row.get("attempts") + out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None + return out + + async def _ingest( c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str] ) -> None: @@ -268,7 +299,11 @@ async def _ingest( try: row = await ingestions.get_ingestion(org, ns, job["ingestion_id"]) assert row is not None - assert await ingest_run.take_and_run(row) == "ready", name + state = await ingest_run.take_and_run(row) + assert state == "ready", ( + name, + await _ingest_failure_context(org, ns, job["ingestion_id"]), + ) finally: await ch.disconnect() diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index 7bc066acc1..c48beb3131 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -113,6 +113,31 @@ async def landed() -> bool: await eventually(landed, description="the org's deliveries published") +async def test_digest_source_proof_reads_exact_authoritative_metadata( + call: Any, org: str, pg_conn: Any +) -> None: + from server.core import pg + from server.memory.store import pg_memory + + ns = f"pg-{uuid.uuid4().hex[:8]}" + response = await call("memory.set", _set(ns, "source", "a source fact")) + assert response.status_code == 200, response.text + key = (org, ns, "a", "-", "-", "source") + other_key = (org, ns, "another-agent", "-", "-", "source") + stored = await pg_conn.fetchval( + "SELECT row FROM loom.memory_current WHERE org = $1 AND namespace = $2", org, ns + ) + try: + heads = await pg_memory.summary_source_heads([key, other_key]) + decoded = pg_memory.decode_row(stored) + assert heads == { + key: {col: decoded.get(col) for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS} + } + assert "value" not in heads[key] and "value_json" not in heads[key] + finally: + await pg.close_pool() + + @pytest.mark.usefixtures("pg_authority") async def test_a_write_is_listed_and_rolled_back_right_after_it_answers( call: Any, org: str diff --git a/tests/integration/test_scim_person_live_ch.py b/tests/integration/test_scim_person_live_ch.py index 34818825b7..a3eeafaa1b 100644 --- a/tests/integration/test_scim_person_live_ch.py +++ b/tests/integration/test_scim_person_live_ch.py @@ -125,13 +125,17 @@ async def people(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[People]: query_all, query_one = client.query_all, client.query_one async def all_rows(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_all(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer async def one_row(sql: str, params: dict[str, Any] | None = None, **kwargs: Any) -> Any: + observing_person = ch.current_query_tag() == "auth.scim_people" answer = await query_one(sql, params=params, **kwargs) - made.results.append((sql, ch.statement_settings(sql), answer)) + if observing_person: + made.results.append((sql, ch.statement_settings(sql), answer)) return answer monkeypatch.setattr(client, "query_all", all_rows) diff --git a/tests/unit/test_cell_attach_cleanup.py b/tests/unit/test_cell_attach_cleanup.py index ec359f3d40..88df8ee92b 100644 --- a/tests/unit/test_cell_attach_cleanup.py +++ b/tests/unit/test_cell_attach_cleanup.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import pytest_asyncio from server.cli import cell_attach, cell_control from tests.unit.cell_control_owner import owner @@ -96,7 +97,8 @@ async def work(): await asyncio.gather(*owned_tasks, return_exceptions=True) -@pytest.mark.asyncio +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance(): closing = asyncio.Event() conn = AsyncMock() @@ -152,3 +154,71 @@ async def open(owned, what): if failed: assert caught.value is original conn.close.assert_awaited_once() + + +@pytest.mark.parametrize("scheduler_s", [0.0, 0.16]) +@pytest.mark.asyncio(loop_scope="function") +@pytest.mark.usefixtures("_cleanup_clock") +async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s): + import time as wall_time + + real_monotonic = wall_time.monotonic + elapsed = 0.0 + + def monotonic(): + return real_monotonic() + elapsed + + actual_close = cell_control.Owner.close + + async def close(owned): + nonlocal elapsed + try: + return await actual_close(owned) + finally: + elapsed += scheduler_s + + monkeypatch.setattr(wall_time, "monotonic", monotonic) + monkeypatch.setattr(cell_control.Owner, "close", close) + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + + +@pytest_asyncio.fixture(loop_scope="function") +async def _cleanup_clock(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + before = set(asyncio.all_tasks()) + loop = asyncio.get_running_loop() + clock = ManualClock(t=loop.time()) + selector = loop._selector + real_select = selector.select + + def select(timeout=None): + if timeout is not None: + clock.advance(max(0.0, timeout)) + timeout = 0 + return real_select(timeout) + + # The cleanup protocol spends idle time, independently of CPU scheduling. + monkeypatch.setattr(loop, "time", clock.monotonic) + monkeypatch.setattr(selector, "select", select) + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + monkeypatch.setattr(f"{__name__}.time", clock) + yield clock + leaked = asyncio.all_tasks() - before - {asyncio.current_task()} + for task in leaked: + task.cancel() + await asyncio.gather(*leaked, return_exceptions=True) + assert not leaked, "cell cleanup leaves no task on its private loop" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock): + from server.core import pg + + loop = asyncio.get_running_loop() + assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic + began = _cleanup_clock.monotonic() + await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance() + assert _cleanup_clock.monotonic() - began == pytest.approx(0.03) diff --git a/tests/unit/test_cell_trigger_default_comparison.py b/tests/unit/test_cell_trigger_default_comparison.py new file mode 100644 index 0000000000..4f712913a7 --- /dev/null +++ b/tests/unit/test_cell_trigger_default_comparison.py @@ -0,0 +1,66 @@ +"""Default-field proof checks keep their refusals without per-field JSON setup.""" + +from copy import deepcopy + +import pytest + +from tests.unit.test_cell_trigger_ci_creator import keeper_config_case as keeper_config_case +from tests.unit.test_cell_trigger_ci_creator import protocol_reader as protocol_reader +from tests.unit.test_cell_trigger_provenance_reuse import _case + + +def _expected(native, specification, context, row, audit, api): + native.Codec(specification).expected( + row, + audit, + context, + api=api, + metadata={"summary": {"jti": "b" * 32}}, + driver={"host": "proof-host", "pid": 701}, + ) + + +@pytest.mark.parametrize("api", [False, True], ids=["delivery", "api"]) +def test_default_proof_bounds_json_encoder_setup( + keeper_config_case, protocol_reader, monkeypatch, api +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, api) + calls = 0 + original = native.json.dumps + + def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(native.json, "dumps", counted) + _expected(native, specification, context, memories[0], audits[0], api) + # One row has 43 defaults; encoder setup must not repeat for each field. + assert calls <= 16 + + +@pytest.mark.parametrize("index", range(43)) +def test_default_proof_refuses_each_changed_field(keeper_config_case, protocol_reader, index): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + field, default = list(specification["sparse_defaults"].items())[index] + assert default != {"unexpected": True} + row = deepcopy(memories[0]) + row[field] = {"unexpected": True} + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) + + +@pytest.mark.parametrize("replacement", [False, 0.0]) +def test_default_proof_keeps_json_numeric_type_refusals( + keeper_config_case, protocol_reader, replacement +): + native, *_ = keeper_config_case + specification, context, _, _, memories, audits, _ = _case(native, 1, False) + assert specification["sparse_defaults"]["pinned"] == 0 + row = deepcopy(memories[0]) + row["pinned"] = replacement + assert row["pinned"] == 0 + with pytest.raises(native.Refused): + _expected(native, specification, context, row, audits[0], False) diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py new file mode 100644 index 0000000000..fcb1ef19b4 --- /dev/null +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -0,0 +1,117 @@ +"""The retrieval fixture exposes ingestion state without printing stored content.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import pytest + +from server.core import ch +from server.knowledge.app import ingest_run +from server.knowledge.store import ingestions +from tests.integration import test_knowledge_retrieval_floors as native + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("observed", "expected"), + [ + ( + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + ), + ( + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + ), + (None, {"observation": "missing"}), + ( + {"state": "secret", "stage": ["secret"], "error_code": "secret", "attempts": True}, + {"state": "unknown", "stage": "unknown", "error_code": "other", "attempts": None}, + ), + ( + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": -1}, + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": None}, + ), + (RuntimeError("secret"), {"observation": "unavailable"}), + ], +) +async def test_failed_ingestion_reads_bounded_status_once( + monkeypatch: pytest.MonkeyPatch, observed: Any, expected: dict[str, Any] +) -> None: + calls = await _install(monkeypatch, "pending", observed) + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected)) + assert "secret" not in str(caught.value) + assert calls == ["connect", "read", "run", "read", "disconnect"] + + +@pytest.mark.asyncio +async def test_ready_ingestion_does_not_read_diagnostic_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "ready", RuntimeError("must not read")) + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert calls == ["connect", "read", "run", "disconnect"] + + +async def _install(monkeypatch: pytest.MonkeyPatch, result: str, observed: Any) -> list[str]: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def post(*_: Any, **__: Any) -> httpx.Response: + return httpx.Response(201, json={"upload_id": "fixture-upload"}) + + async def op(*_: Any, **__: Any) -> dict[str, str]: + return {"ingestion_id": "fixture-ingestion"} + + async def get(org: str, namespace: str, ingestion_id: str) -> Any: + assert (org, namespace, ingestion_id) == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + if calls.count("read") == 1: + return {"ingestion_id": "fixture-ingestion"} + if isinstance(observed, Exception): + raise observed + if isinstance(observed, asyncio.Event): + try: + await observed.wait() + finally: + calls.append("observation-cancelled") + if observed is None: + return None + return {**observed, "error_message": "secret", "content": "secret", "metadata": "secret"} + + async def run(row: dict[str, str]) -> str: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return result + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(native, "_post", post) + monkeypatch.setattr(native, "_op", op) + monkeypatch.setattr(ingestions, "get_ingestion", get) + monkeypatch.setattr(ingest_run, "take_and_run", run) + return calls + + +@pytest.mark.asyncio +async def test_blocked_status_read_does_not_hide_the_ingestion_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "pending", asyncio.Event()) + with pytest.raises(AssertionError) as caught: + await asyncio.wait_for( + native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []), + timeout=2, + ) + assert "unavailable" in str(caught.value) + assert calls == ["connect", "read", "run", "read", "observation-cancelled", "disconnect"] diff --git a/tests/unit/test_pg_durable_reads.py b/tests/unit/test_pg_durable_reads.py index 80e42bfc6e..61ac2db25a 100644 --- a/tests/unit/test_pg_durable_reads.py +++ b/tests/unit/test_pg_durable_reads.py @@ -112,6 +112,7 @@ "server/memory/store/records.py": "a retirement reads its writers durably; " "`live_writer_record` filters a request's read, which commits through `memory.set`", "server/memory/store/redaction.py": "decides a redaction, which commits", + "server/memory/store/summary_sources.py": "filters which digests a search serves; sends nothing", "server/memory/workers/pg_payload_clear.py": "clears through its own commit, and only " "revisions delivered a horizon ago that ClickHouse holds", } diff --git a/tests/unit/test_scim_person_query_observation.py b/tests/unit/test_scim_person_query_observation.py new file mode 100644 index 0000000000..a0f9ef7f06 --- /dev/null +++ b/tests/unit/test_scim_person_query_observation.py @@ -0,0 +1,116 @@ +"""Person-query evidence excludes reads from independent background tasks.""" + +from __future__ import annotations + +import asyncio +from contextvars import Context +from typing import Any + +import pytest + +from server.auth import scim_people +from server.core import ch, rmt_carry +from server.settings import settings +from tests.integration.test_scim_person_live_ch import people as person_fixture +from tests.unit.test_ch_client_hygiene import _client, _FakeAsync + + +class Queries: + def __init__(self) -> None: + self.started = asyncio.Event() + self.background_done = asyncio.Event() + self.block_person = False + self.transport: list[tuple[str, dict[str, Any]]] = [] + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def _read(self, sql: str) -> dict[str, Any]: + self.transport.append((sql, ch.statement_settings(sql, cls="read"))) + if sql == "SELECT 1 AS person" and self.block_person: + self.started.set() + await self.background_done.wait() + return {"n": 0, "at": None, "altered": None} + + async def query_one(self, sql: str, params: Any = None) -> dict[str, Any]: + return await self._read(sql) + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + return [await self._read(sql)] + + +@pytest.fixture +async def recording(monkeypatch): + client = Queries() + monkeypatch.setattr(ch, "get_client", lambda: client) + stream = person_fixture.__wrapped__(monkeypatch) + observed = await anext(stream) + try: + yield observed, client + finally: + await stream.aclose() + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_background_schema_reads_do_not_enter_person_evidence( + recording, monkeypatch, read_timeout, many +): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", read_timeout) + client.block_person = True + + async def background(): + await client.started.wait() + try: + with ch.query_tag("rmt_carry.columns"): + await rmt_carry._schema_stamp(client) + finally: + client.background_done.set() + + watcher = asyncio.create_task(background(), context=Context()) + try: + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + await watcher + finally: + if not watcher.done(): + watcher.cancel() + await asyncio.gather(watcher, return_exceptions=True) + + assert len(client.transport) == 2 + assert any("system.tables" in sql for sql, _ in client.transport) + assert [sql for sql, _, _ in observed.results] == ["SELECT 1 AS person"] + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("many", [False, True]) +async def test_unbounded_person_reads_remain_visible(recording, monkeypatch, many): + observed, client = recording + monkeypatch.setattr(settings, "ch_timeout_read_s", 0) + with ch.query_tag("auth.scim_people"): + query = client.query_all if many else client.query_one + await query("SELECT 1 AS person") + assert len(observed.results) == 1 + with pytest.raises(KeyError, match="max_execution_time"): + assert 0 < observed.results[0][1]["max_execution_time"] <= 1.75 + + +@pytest.mark.parametrize("read_timeout", [0, 2.5]) +@pytest.mark.parametrize("many", [False, True]) +async def test_actual_person_reader_applies_its_finite_budget_to_driver( + monkeypatch, read_timeout, many +): + driver = _FakeAsync() + client = _client(monkeypatch, driver, ch_timeout_read_s=read_timeout) + monkeypatch.setattr(ch, "get_client", lambda: client) + decision = scim_people._Decision("org", "owner", "owner@example.test", scim_people._now() + 10) + with ch.statement_timeout(1.75), ch.query_tag("auth.scim_people"): + await decision.query("SELECT 1 AS person", {}, many=many) + calls = [applied for kind, sql, applied in driver.calls if sql == "SELECT 1 AS person"] + assert len(calls) == 1 + assert 0 < calls[0]["max_execution_time"] <= 1.75 diff --git a/tests/unit/test_summary_source_authority.py b/tests/unit/test_summary_source_authority.py new file mode 100644 index 0000000000..5a15679ec9 --- /dev/null +++ b/tests/unit/test_summary_source_authority.py @@ -0,0 +1,295 @@ +"""A parked source publication cannot make an obsolete digest readable.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.core import pg +from server.memory.app.retrieval.summaries import fetch_session_summaries +from server.memory.store import pg_memory +from server.memory.store.rows import identity_key + +KEY = ("org", "namespace", "writer", "reader", "session", "source") +STAMP = datetime(2026, 1, 1) + + +def _source() -> dict[str, Any]: + return dict( + zip(("org", "namespace", "agent", "user_id", "session_id", "memory_key"), KEY, strict=True), + version=100, + content_sha256="a" * 64, + status="active", + sensitivity="low", + tombstoned_at=None, + redacted_at=None, + expires_at=None, + valid_to=None, + created_at=STAMP, + valid_at=None, + temporal_anchor=None, + invalid_at=None, + ) + + +class SourceProjection: + def __init__(self, authority: dict[Any, Any], replacement: dict[str, Any] | None) -> None: + self.authority = authority + self.replacement = replacement + self.lineage = ["source"] + self.projected_hash: object = "a" * 64 + + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if "FROM loom.session_summaries FINAL" in sql: + return [ + { + "agent": "writer", + "user_id": "reader", + "session_id": "session", + "version": 200, + "derived_from": self.lineage, + "summary_text": "a source fact", + "distance": 0.1, + } + ] + assert "FROM loom.memory FINAL" in sql + # The source moves after admission and the projection read. Its delivery stays parked. + self.authority.clear() + if self.replacement is not None: + self.authority[identity_key(self.replacement)] = self.replacement + return [ + _source() + | { + "source_user": "reader", + "source_session": "session", + "source_served": True, + "content_sha256": self.projected_hash, + } + ] + + +@pytest.fixture +def source_read(monkeypatch: pytest.MonkeyPatch): + authority = {KEY: _source()} + requested: list[Any] = [] + unavailable = False + + async def read(sql: str, *args: Any, **kwargs: Any) -> list[dict[str, Any]]: + if unavailable: + raise TimeoutError("authority unavailable") + keys = list(zip(*args, strict=True)) + requested.extend(keys) + return [{"row": authority[key]} for key in keys if key in authority] + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg_memory, "_read", read) + + async def retrieve( + replacement: dict[str, Any] | None, + *, + lineage: bool = True, + fail=False, + projected_hash: object = "a" * 64, + ): + nonlocal unavailable + unavailable = fail + ch = SourceProjection(authority, replacement) + ch.projected_hash = projected_hash + if not lineage: + ch.lineage = [] + return await fetch_session_summaries( + ch, + org="org", + namespace="namespace", + agent="writer", + user_id="reader", + query_vec=[1.0, 0.0], + embedding_model="test-model", + ) + + return retrieve, requested + + +@pytest.mark.parametrize( + "change", + [ + {"tombstoned_at": STAMP, "status": "tombstoned"}, + {"version": 150}, + {"content_sha256": "b" * 64}, + {"content_sha256": ""}, + {"status": "rejected"}, + {"sensitivity": "high"}, + {"redacted_at": STAMP}, + {"expires_at": STAMP}, + {"valid_to": STAMP}, + ], + ids=[ + "deleted", + "newer-version", + "same-version-rewrite", + "missing-proof", + "rejected", + "high", + "redacted", + "expired", + "invalid", + ], +) +async def test_a_digest_withholds_a_source_changed_after_the_projection_read(source_read, change): + retrieve, _ = source_read + assert await retrieve(_source() | change) == [] + + +@pytest.mark.parametrize( + "axis", ["org", "namespace", "agent", "user_id", "session_id", "memory_key"] +) +async def test_another_identitys_source_cannot_validate_a_digest(source_read, axis): + retrieve, requested = source_read + assert await retrieve(_source() | {axis: "other"}) == [] + assert requested == [KEY] + + +async def test_a_missing_authoritative_source_withholds_the_digest(source_read): + retrieve, _ = source_read + assert await retrieve(None) == [] + + +async def test_an_unavailable_authority_never_falls_back_to_the_projection(source_read): + retrieve, _ = source_read + assert await retrieve(_source(), fail=True) == [] + + +async def test_the_matching_source_keeps_its_digest_with_the_full_identity(source_read): + retrieve, requested = source_read + head = _source() | {"created_at": STAMP.replace(tzinfo=UTC).astimezone(UTC)} + [hit] = await retrieve(head) + assert hit.content_excerpt == "a source fact" + assert hit.derived_from == ["source"] + assert requested == [KEY] + + +async def test_missing_lineage_remains_transcript_evidence_without_an_authority_read(source_read): + retrieve, requested = source_read + [hit] = await retrieve(None, lineage=False, fail=True) + assert hit.content_excerpt == "a source fact" + assert requested == [] + + +async def test_clickhouse_authority_uses_its_own_source_state(source_read, monkeypatch): + retrieve, requested = source_read + monkeypatch.setattr(pg, "enabled", lambda: False) + [hit] = await retrieve(None, fail=True) + assert hit.content_excerpt == "a source fact" + assert requested == [] + + +async def test_source_proof_reads_metadata_and_restores_sparse_defaults(monkeypatch): + source = _source() + stored = { + col: value + for col, value in source.items() + if col not in pg_memory.SPARSE_DEFAULTS or value != pg_memory.SPARSE_DEFAULTS[col] + } + stored["created_at"] = STAMP.isoformat() + + async def read(sql, *args, durable): + assert list(zip(*args, strict=True)) == [KEY] + assert not durable + assert "USING (org, namespace, agent, user_id, session_id, memory_key)" in sql + assert "SELECT c.row " not in sql + assert "c.row->'value'" not in sql and "c.row->'value_json'" not in sql + assert "jsonb_strip_nulls" in sql + return [{"row": stored}] + + monkeypatch.setattr(pg_memory, "_read", read) + heads = await pg_memory.summary_source_heads([KEY]) + assert heads[KEY] == {col: source[col] for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS} + + +async def test_an_unstorable_source_key_never_reaches_the_authority(monkeypatch): + async def unreadable(*args, **kwargs): + raise AssertionError("an unstorable key must not reach Postgres") + + monkeypatch.setattr(pg_memory, "_read", unreadable) + assert await pg_memory.summary_source_heads([]) == {} + assert await pg_memory.summary_source_heads([(*KEY[:5], "bad\x00key")]) == {} + + +@pytest.mark.parametrize("projected_hash", [b"a" * 64, bytearray(b"a" * 64)]) +async def test_native_hash_bytes_keep_a_matching_authoritative_digest(source_read, projected_hash): + retrieve, requested = source_read + [hit] = await retrieve(_source(), projected_hash=projected_hash) + assert hit.content_excerpt == "a source fact" + assert hit.derived_from == ["source"] + assert requested == [KEY] + + +@pytest.mark.parametrize( + ("projected_hash", "authoritative_hash"), + [ + (b"b" * 64, "a" * 64), + (b"", ""), + (b"\x00" * 64, ""), + (b"g" * 64, "g" * 64), + (b"\xff" * 64, "\ufffd" * 64), + ], + ids=["changed", "empty", "null-padding", "nonhex", "invalid-utf8"], +) +async def test_native_hash_bytes_cannot_validate_changed_or_invalid_sources( + source_read, projected_hash, authoritative_hash +): + retrieve, requested = source_read + assert ( + await retrieve( + _source() | {"content_sha256": authoritative_hash}, projected_hash=projected_hash + ) + == [] + ) + assert requested == [KEY] + + +@pytest.mark.parametrize("outcome", ["normal", "helper-error", "assertion-error"]) +async def test_digest_metadata_integration_check_closes_its_pool(monkeypatch, outcome): + from tests.integration import test_pg_authority + + stored = _source() + events = [] + + async def call(op, body): + assert op == "memory.set" + stored["namespace"] = body["namespace"] + stored["agent"] = body["agent"] + stored["user_id"] = stored["session_id"] = "-" + return SimpleNamespace(status_code=200, text="") + + async def fetchval(sql, org, namespace): + assert (org, namespace) == (stored["org"], stored["namespace"]) + return stored + + async def heads(keys): + events.append("opened") + assert keys[0] == identity_key(stored) + if outcome == "helper-error": + raise RuntimeError("source metadata unavailable") + if outcome == "assertion-error": + return {} + return {keys[0]: {col: stored.get(col) for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS}} + + async def close_pool(): + assert events == ["opened"] + events.append("closed") + + monkeypatch.setattr(pg_memory, "summary_source_heads", heads) + monkeypatch.setattr(pg, "close_pool", close_pool) + check = test_pg_authority.test_digest_source_proof_reads_exact_authoritative_metadata + args = (call, stored["org"], SimpleNamespace(fetchval=fetchval)) + if outcome == "normal": + await check(*args) + else: + error = RuntimeError if outcome == "helper-error" else AssertionError + with pytest.raises(error): + await check(*args) + assert events == ["opened", "closed"]