diff --git a/tests/integration/test_knowledge_retrieval_floors.py b/tests/integration/test_knowledge_retrieval_floors.py index 10bd835305..3ff0b402d8 100644 --- a/tests/integration/test_knowledge_retrieval_floors.py +++ b/tests/integration/test_knowledge_retrieval_floors.py @@ -235,6 +235,37 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]: return dict(r.json()) +async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]: + """Read only fixed status fields; a retry message can contain source content.""" + from server.knowledge.store import ingestions + + try: + async with asyncio.timeout(1): + row = await ingestions.get_ingestion(org, ns, ingestion_id) + except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion + return {"observation": "unavailable"} + if row is None: + return {"observation": "missing"} + choices = { + "state": {"pending", "processing", "ready", "partial", "failed", "cancelled"}, + "stage": {"", "parse", "embed", "publish"}, + "error_code": {"", "retrying", "retries_exhausted", "revision_conflict"}, + } + out: dict[str, Any] = {} + for key, allowed in choices.items(): + value = row.get(key) + out[key] = ( + value + if type(value) is str and value in allowed + else "other" + if key == "error_code" + else "unknown" + ) + attempts = row.get("attempts") + out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None + return out + + async def _ingest( c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str] ) -> None: @@ -268,7 +299,11 @@ async def _ingest( try: row = await ingestions.get_ingestion(org, ns, job["ingestion_id"]) assert row is not None - assert await ingest_run.take_and_run(row) == "ready", name + state = await ingest_run.take_and_run(row) + assert state == "ready", ( + name, + await _ingest_failure_context(org, ns, job["ingestion_id"]), + ) finally: await ch.disconnect() diff --git a/tests/unit/test_knowledge_retrieval_failure_context.py b/tests/unit/test_knowledge_retrieval_failure_context.py new file mode 100644 index 0000000000..fcb1ef19b4 --- /dev/null +++ b/tests/unit/test_knowledge_retrieval_failure_context.py @@ -0,0 +1,117 @@ +"""The retrieval fixture exposes ingestion state without printing stored content.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import pytest + +from server.core import ch +from server.knowledge.app import ingest_run +from server.knowledge.store import ingestions +from tests.integration import test_knowledge_retrieval_floors as native + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("observed", "expected"), + [ + ( + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + {"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1}, + ), + ( + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + {"state": "pending", "stage": "publish", "error_code": "", "attempts": 0}, + ), + (None, {"observation": "missing"}), + ( + {"state": "secret", "stage": ["secret"], "error_code": "secret", "attempts": True}, + {"state": "unknown", "stage": "unknown", "error_code": "other", "attempts": None}, + ), + ( + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": -1}, + {"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": None}, + ), + (RuntimeError("secret"), {"observation": "unavailable"}), + ], +) +async def test_failed_ingestion_reads_bounded_status_once( + monkeypatch: pytest.MonkeyPatch, observed: Any, expected: dict[str, Any] +) -> None: + calls = await _install(monkeypatch, "pending", observed) + with pytest.raises(AssertionError) as caught: + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected)) + assert "secret" not in str(caught.value) + assert calls == ["connect", "read", "run", "read", "disconnect"] + + +@pytest.mark.asyncio +async def test_ready_ingestion_does_not_read_diagnostic_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "ready", RuntimeError("must not read")) + await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []) + assert calls == ["connect", "read", "run", "disconnect"] + + +async def _install(monkeypatch: pytest.MonkeyPatch, result: str, observed: Any) -> list[str]: + calls: list[str] = [] + + class Client: + async def connect(self) -> None: + calls.append("connect") + + async def disconnect(self) -> None: + calls.append("disconnect") + + async def post(*_: Any, **__: Any) -> httpx.Response: + return httpx.Response(201, json={"upload_id": "fixture-upload"}) + + async def op(*_: Any, **__: Any) -> dict[str, str]: + return {"ingestion_id": "fixture-ingestion"} + + async def get(org: str, namespace: str, ingestion_id: str) -> Any: + assert (org, namespace, ingestion_id) == ("fixture-org", "fixture-ns", "fixture-ingestion") + calls.append("read") + if calls.count("read") == 1: + return {"ingestion_id": "fixture-ingestion"} + if isinstance(observed, Exception): + raise observed + if isinstance(observed, asyncio.Event): + try: + await observed.wait() + finally: + calls.append("observation-cancelled") + if observed is None: + return None + return {**observed, "error_message": "secret", "content": "secret", "metadata": "secret"} + + async def run(row: dict[str, str]) -> str: + assert row == {"ingestion_id": "fixture-ingestion"} + calls.append("run") + return result + + monkeypatch.setattr(ch, "get_client", Client) + monkeypatch.setattr(native, "_post", post) + monkeypatch.setattr(native, "_op", op) + monkeypatch.setattr(ingestions, "get_ingestion", get) + monkeypatch.setattr(ingest_run, "take_and_run", run) + return calls + + +@pytest.mark.asyncio +async def test_blocked_status_read_does_not_hide_the_ingestion_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = await _install(monkeypatch, "pending", asyncio.Event()) + with pytest.raises(AssertionError) as caught: + await asyncio.wait_for( + native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []), + timeout=2, + ) + assert "unavailable" in str(caught.value) + assert calls == ["connect", "read", "run", "read", "observation-cancelled", "disconnect"]