Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion tests/integration/test_knowledge_retrieval_floors.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,37 @@ async def _op(c: httpx.AsyncClient, op: str, **body: Any) -> dict[str, Any]:
return dict(r.json())


async def _ingest_failure_context(org: str, ns: str, ingestion_id: str) -> dict[str, Any]:
"""Read only fixed status fields; a retry message can contain source content."""
from server.knowledge.store import ingestions

try:
async with asyncio.timeout(1):
row = await ingestions.get_ingestion(org, ns, ingestion_id)
except Exception: # noqa: BLE001 — diagnostic reads must not replace the failed assertion
return {"observation": "unavailable"}
if row is None:
return {"observation": "missing"}
choices = {
"state": {"pending", "processing", "ready", "partial", "failed", "cancelled"},
"stage": {"", "parse", "embed", "publish"},
"error_code": {"", "retrying", "retries_exhausted", "revision_conflict"},
}
out: dict[str, Any] = {}
for key, allowed in choices.items():
value = row.get(key)
out[key] = (
value
if type(value) is str and value in allowed
else "other"
if key == "error_code"
else "unknown"
)
attempts = row.get("attempts")
out["attempts"] = attempts if type(attempts) is int and 0 <= attempts <= 1_000_000 else None
return out


async def _ingest(
c: httpx.AsyncClient, org: str, ns: str, name: str, data: bytes, tags: list[str]
) -> None:
Expand Down Expand Up @@ -268,7 +299,11 @@ async def _ingest(
try:
row = await ingestions.get_ingestion(org, ns, job["ingestion_id"])
assert row is not None
assert await ingest_run.take_and_run(row) == "ready", name
state = await ingest_run.take_and_run(row)
assert state == "ready", (
name,
await _ingest_failure_context(org, ns, job["ingestion_id"]),
)
finally:
await ch.disconnect()

Expand Down
117 changes: 117 additions & 0 deletions tests/unit/test_knowledge_retrieval_failure_context.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
"""The retrieval fixture exposes ingestion state without printing stored content."""

from __future__ import annotations

import asyncio
from typing import Any

import httpx
import pytest

from server.core import ch
from server.knowledge.app import ingest_run
from server.knowledge.store import ingestions
from tests.integration import test_knowledge_retrieval_floors as native


@pytest.mark.asyncio
@pytest.mark.parametrize(
("observed", "expected"),
[
(
{"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1},
{"state": "pending", "stage": "embed", "error_code": "retrying", "attempts": 1},
),
(
{"state": "pending", "stage": "publish", "error_code": "", "attempts": 0},
{"state": "pending", "stage": "publish", "error_code": "", "attempts": 0},
),
(None, {"observation": "missing"}),
(
{"state": "secret", "stage": ["secret"], "error_code": "secret", "attempts": True},
{"state": "unknown", "stage": "unknown", "error_code": "other", "attempts": None},
),
(
{"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": -1},
{"state": "failed", "stage": "", "error_code": "retries_exhausted", "attempts": None},
),
(RuntimeError("secret"), {"observation": "unavailable"}),
],
)
async def test_failed_ingestion_reads_bounded_status_once(
monkeypatch: pytest.MonkeyPatch, observed: Any, expected: dict[str, Any]
) -> None:
calls = await _install(monkeypatch, "pending", observed)
with pytest.raises(AssertionError) as caught:
await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", [])
assert str(caught.value).split("\n", 1)[0] == repr(("regions.csv", expected))
assert "secret" not in str(caught.value)
assert calls == ["connect", "read", "run", "read", "disconnect"]


@pytest.mark.asyncio
async def test_ready_ingestion_does_not_read_diagnostic_status(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = await _install(monkeypatch, "ready", RuntimeError("must not read"))
await native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", [])
assert calls == ["connect", "read", "run", "disconnect"]


async def _install(monkeypatch: pytest.MonkeyPatch, result: str, observed: Any) -> list[str]:
calls: list[str] = []

class Client:
async def connect(self) -> None:
calls.append("connect")

async def disconnect(self) -> None:
calls.append("disconnect")

async def post(*_: Any, **__: Any) -> httpx.Response:
return httpx.Response(201, json={"upload_id": "fixture-upload"})

async def op(*_: Any, **__: Any) -> dict[str, str]:
return {"ingestion_id": "fixture-ingestion"}

async def get(org: str, namespace: str, ingestion_id: str) -> Any:
assert (org, namespace, ingestion_id) == ("fixture-org", "fixture-ns", "fixture-ingestion")
calls.append("read")
if calls.count("read") == 1:
return {"ingestion_id": "fixture-ingestion"}
if isinstance(observed, Exception):
raise observed
if isinstance(observed, asyncio.Event):
try:
await observed.wait()
finally:
calls.append("observation-cancelled")
if observed is None:
return None
return {**observed, "error_message": "secret", "content": "secret", "metadata": "secret"}

async def run(row: dict[str, str]) -> str:
assert row == {"ingestion_id": "fixture-ingestion"}
calls.append("run")
return result

monkeypatch.setattr(ch, "get_client", Client)
monkeypatch.setattr(native, "_post", post)
monkeypatch.setattr(native, "_op", op)
monkeypatch.setattr(ingestions, "get_ingestion", get)
monkeypatch.setattr(ingest_run, "take_and_run", run)
return calls


@pytest.mark.asyncio
async def test_blocked_status_read_does_not_hide_the_ingestion_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = await _install(monkeypatch, "pending", asyncio.Event())
with pytest.raises(AssertionError) as caught:
await asyncio.wait_for(
native._ingest(None, "fixture-org", "fixture-ns", "regions.csv", b"csv", []),
timeout=2,
)
assert "unavailable" in str(caught.value)
assert calls == ["connect", "read", "run", "read", "observation-cancelled", "disconnect"]