diff --git a/tests/integration/test_knowledge_move.py b/tests/integration/test_knowledge_move.py index c7dbe82385..5e7e2258c2 100644 --- a/tests/integration/test_knowledge_move.py +++ b/tests/integration/test_knowledge_move.py @@ -616,10 +616,62 @@ async def test_a_retry_onto_a_copy_with_its_chunks_and_tables_gone_copies_it_aga async def cut_off(doc: Any) -> None: raise RuntimeError("cut off before the delete") + from server.core.ch import CHClient, error_code + from server.core.rmt_carry import fingerprint + + send_failure: dict[str, Any] | None = None + original_insert = CHClient._fenced_insert + + async def observed_insert( + client: Any, + table: str, + data: Any, + columns: Any, + types: Any, + stored: frozenset[str] | None, + settings: Any, + deadline: float, + ) -> Any: + nonlocal send_failure + try: + return await original_insert( + client, table, data, columns, types, stored, settings, deadline + ) + except Exception as error: + if table == "loom.datasets": + kind = type(error).__name__ + try: + code = error_code(error) + except Exception: # noqa: BLE001 — A defective formatter must not replace the send failure. + code = None + send_failure = { + "kind": kind + if kind + in { + "DatabaseError", + "OperationalError", + "BackendUnavailableError", + "TimeoutError", + } + else "other", + "code": code if code is not None and 0 <= code <= 65535 else None, + "stored_count": None if stored is None else len(stored), + "stored_fingerprint": None if stored is None else fingerprint(stored), + } + raise + with monkeypatch.context() as m: m.setattr(move, "_delete_source", cut_off) + m.setattr(CHClient, "_fenced_insert", observed_insert) with pytest.raises(RuntimeError): - await _move_here(move, src, dst, uri) + try: + await _move_here(move, src, dst, uri) + except Exception as error: + if send_failure is not None: + error.add_note( + "catalog send failure: " + json.dumps(send_failure, sort_keys=True) + ) + raise # What a row insert that landed after its take-back leaves: the row, no chunks, no # catalog, no table rows. for table in ("loom.chunks", "loom.datasets", "loom.dataset_rows"): diff --git a/tests/unit/test_knowledge_move_failure_context.py b/tests/unit/test_knowledge_move_failure_context.py new file mode 100644 index 0000000000..6d883693af --- /dev/null +++ b/tests/unit/test_knowledge_move_failure_context.py @@ -0,0 +1,251 @@ +"""Catalog-copy failures report send metadata without changing their outcome.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from types import SimpleNamespace +from typing import Any + +import pytest +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core.ch import CHClient +from server.core.errors import BackendUnavailableError +from tests.integration import test_knowledge_move as native + +CASE = native.test_a_retry_onto_a_copy_with_its_chunks_and_tables_gone_copies_it_again + + +class UnreadableFailure(Exception): + def __str__(self) -> str: + raise ValueError("private formatter") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "stored", "kind", "code"), + [ + ( + DatabaseError("Code: 395. private-query"), + frozenset({"org", "private-column"}), + "DatabaseError", + 395, + ), + (TimeoutError("private-host"), frozenset(), "TimeoutError", None), + (ValueError("private-content"), None, "other", None), + (UnreadableFailure(), frozenset({"org"}), "other", None), + ( + DatabaseError("Code: 999999999. private-query"), + frozenset({"org"}), + "DatabaseError", + None, + ), + ], +) +@pytest.mark.parametrize("translated", [False, True]) +async def test_catalog_failure_keeps_exception_identity_and_only_fixed_metadata( + monkeypatch: pytest.MonkeyPatch, + error: Exception, + stored: frozenset[str] | None, + kind: str, + code: int | None, + translated: bool, +) -> None: + outward = BackendUnavailableError("public refusal") if translated else error + state = _install(monkeypatch, error=error, outward=outward, stored=stored) + with pytest.raises(type(outward)) as caught: + await CASE("http://fixture.test", state.client, state.move, monkeypatch) + assert caught.value is outward + (note,) = caught.value.__notes__ + assert note.startswith("catalog send failure: ") + assert json.loads(note.removeprefix("catalog send failure: ")) == { + "kind": kind, + "code": code, + "stored_count": None if stored is None else len(stored), + "stored_fingerprint": None + if stored is None + else hashlib.sha256("\n".join(sorted(stored)).encode()).hexdigest()[:16], + } + assert len(note) < 200 + assert "private" not in note + assert state.calls == [ + "agent-enter", + "add", + "agent-exit", + "write-table", + "live", + "move", + "send", + ] + assert CHClient._fenced_insert is state.original_send + + +@pytest.mark.asyncio +async def test_success_keeps_the_cutoff_and_retry_checks_without_extra_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state = _install(monkeypatch) + await CASE("http://fixture.test", state.client, state.move, monkeypatch) + assert state.calls == [ + "agent-enter", + "add", + "agent-exit", + "write-table", + "live", + "move", + "send", + "delete", + "delete", + "delete", + "live-rows", + "move", + "send", + "live", + "live-rows", + "rows", + ] + assert CHClient._fenced_insert is state.original_send + + +@pytest.mark.asyncio +async def test_a_failure_before_the_catalog_send_gains_no_note( + monkeypatch: pytest.MonkeyPatch, +) -> None: + error = ValueError("private pre-send failure") + state = _install(monkeypatch, error=error, before_send=True) + with pytest.raises(ValueError) as caught: + await CASE("http://fixture.test", state.client, state.move, monkeypatch) + assert caught.value is error + assert not getattr(error, "__notes__", []) + assert state.calls[-1] == "move" + + +@pytest.mark.asyncio +async def test_another_tables_failure_gains_no_catalog_note( + monkeypatch: pytest.MonkeyPatch, +) -> None: + error = ValueError("private other-table failure") + state = _install(monkeypatch, error=error, table="loom.documents") + with pytest.raises(ValueError) as caught: + await CASE("http://fixture.test", state.client, state.move, monkeypatch) + assert caught.value is error + assert not getattr(error, "__notes__", []) + + +def _install( + monkeypatch: pytest.MonkeyPatch, + *, + error: Exception | None = None, + outward: Exception | None = None, + stored: frozenset[str] | None = frozenset({"org"}), + before_send: bool = False, + table: str = "loom.datasets", +) -> SimpleNamespace: + calls: list[str] = [] + src, dst, uri = "fixture-source", "fixture-target", "doc://fixture" + sender, data, columns, types, settings = ( + object(), + [["private-data"]], + ["private-column"], + ["String"], + {"private-token": "secret"}, + ) + args = (table, data, columns, types, stored, settings, 123.5) + result = object() + + async def send(self: Any, *forwarded: Any) -> Any: + assert self is sender + assert len(forwarded) == len(args) + assert all(actual is expected for actual, expected in zip(forwarded, args, strict=True)) + calls.append("send") + if error is not None: + raise error + return result + + async def moved(move: Any, source: str, target: str, source_uri: str) -> list[dict[str, Any]]: + assert (source, target, source_uri) == (src, dst, uri) + calls.append("move") + if before_send: + assert error is not None + raise error + try: + got = await CHClient._fenced_insert(sender, *args) + except Exception as caught: + assert caught is error + if outward is not None: + raise outward from None + raise + assert got is result + if calls.count("move") == 1: + await move._delete_source(None) + return [{"source_uri": uri, "moved": True, "reason": ""}] + + class Agent: + def __init__(self, **kwargs: Any) -> None: + assert kwargs == { + "org": native.ORG, + "namespace": src, + "agent": "w", + "base_url": "http://fixture.test", + } + self.knowledge = SimpleNamespace(add=self.add) + + async def add(self, **kwargs: Any) -> None: + assert kwargs == { + "content": "region,total\nnorth,3", + "source_uri": uri, + "title": "sales", + } + calls.append("add") + + async def __aenter__(self) -> Agent: + calls.append("agent-enter") + return self + + async def __aexit__(self, *_: Any) -> None: + calls.append("agent-exit") + + class Client: + def command(self, sql: str, *, parameters: dict[str, str]) -> None: + assert sql == ( + f"DELETE FROM {('loom.chunks', 'loom.datasets', 'loom.dataset_rows')[calls.count('delete')]} " + "WHERE org = {org:String} AND namespace = {ns:String} SETTINGS lightweight_deletes_sync = 2" + ) + assert parameters == {"org": native.ORG, "ns": dst} + calls.append("delete") + + def query(self, sql: str, *, parameters: dict[str, str]) -> Any: + assert sql == ( + "SELECT s1 FROM loom.dataset_rows WHERE org = {org:String} AND namespace = {ns:String} ORDER BY row_id" + ) + assert parameters == {"org": native.ORG, "ns": dst} + calls.append("rows") + return SimpleNamespace(result_rows=[("north",), ("south",)]) + + def live(client: Any, namespace: str, source_uri: str) -> list[str]: + assert client is instance and source_uri == uri + assert namespace == (src if calls.count("move") == 0 else dst) + calls.append("live") + return ["fixture-chunk"] + + def live_rows(client: Any, namespace: str, source_uri: str) -> int: + assert client is instance and source_uri == uri + calls.append("live-rows") + return 1 if namespace == dst else 0 + + def write(client: Any, namespace: str, source_uri: str) -> None: + assert client is instance and (namespace, source_uri) == (src, uri) + calls.append("write-table") + + instance = Client() + move = SimpleNamespace(_delete_source=None) + monkeypatch.setitem(sys.modules, "loom", SimpleNamespace(Agent=Agent)) + monkeypatch.setattr(CHClient, "_fenced_insert", send) + monkeypatch.setattr(native, "_places", lambda: (src, dst, uri)) + monkeypatch.setattr(native, "_write_table", write) + monkeypatch.setattr(native, "_live", live) + monkeypatch.setattr(native, "_live_rows", live_rows) + monkeypatch.setattr(native, "_move_here", moved) + return SimpleNamespace(calls=calls, client=instance, move=move, original_send=send)