diff --git a/server/core/object_store.py b/server/core/object_store.py
index a28f3668e7..e032ae43a5 100644
--- a/server/core/object_store.py
+++ b/server/core/object_store.py
@@ -170,29 +170,49 @@ def new_location(org: str, namespace: str, sha: str) -> Location:
@functools.lru_cache(maxsize=4)
-def _client_for(endpoint: str, region: str, connect_s: float, read_s: float) -> Any:
+def _client_for(
+ endpoint: str, region: str, connect_s: float, read_s: float, *, diagnostic: bool = False
+) -> Any:
"""A client for the endpoint, region and timeouts, cached. Each build makes its own
session; the lock serialises building the session and the client, as botocore
loads its data files and credential chain then and is not reliably safe to do
so in two threads at once. The cache does not hold the lock, so two first
calls may each build a client and one is kept."""
+ if diagnostic:
+ from server.core import object_store_probe
+
+ object_store_probe.require_sdk()
import boto3
- config = _client_config(connect_s, read_s)
+ config = _client_config(connect_s, read_s, diagnostic=diagnostic)
with _CLIENT_LOCK:
session = boto3.session.Session()
- return session.client(
+ client = session.client(
"s3", endpoint_url=endpoint or None, region_name=region or None, config=config
)
+ if diagnostic:
+ try:
+ object_store_probe.install(client)
+ except BaseException:
+ try:
+ client.close()
+ except Exception: # noqa: BLE001 — retain the diagnostic client refusal
+ pass
+ raise
+ return client
-def _client_config(connect_s: float, read_s: float) -> Any:
+def _client_config(connect_s: float, read_s: float, *, diagnostic: bool = False) -> Any:
from botocore.config import Config
return Config(
connect_timeout=connect_s,
read_timeout=read_s,
- retries={"max_attempts": 3, "mode": "standard"},
+ retries=(
+ {"total_max_attempts": 1, "mode": "standard"}
+ if diagnostic
+ else {"max_attempts": 3, "mode": "standard"}
+ ),
max_pool_connections=_THREADS, # one connection per bucket thread
# Checksums only where a request needs one (`put` sends its SHA-256 itself):
# boto3's default of one on every request is refused by some S3-compatible stores.
diff --git a/server/core/object_store_probe.py b/server/core/object_store_probe.py
new file mode 100644
index 0000000000..6acad3c0fd
--- /dev/null
+++ b/server/core/object_store_probe.py
@@ -0,0 +1,565 @@
+"""Bounded reads and exact-owned marker writes for cell separation checks.
+
+These operations use the deployment's ordinary bucket configuration. A refused
+or unknown response is never evidence of absence or completed cleanup.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import os
+import re
+import stat
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from email.utils import format_datetime, parsedate_to_datetime
+from importlib import metadata
+from pathlib import Path
+from typing import Any, cast
+
+from server.core import object_store
+from server.core.cell import NAME
+from server.core.ch import set_query_tag
+from server.settings import settings
+
+_LIST_BYTES = 8 * 1024 * 1024
+_ERROR_BYTES = 64 * 1024
+_MARKER_BYTES = 1024
+_CHUNK = 64 * 1024
+_ETAG = re.compile(r'"[\x21\x23-\x7e]{1,1022}"')
+_SENT = "loom_cell_probe_sent"
+_MODEL_SHA256 = "fef1a4b5a29d993ec1659fd203d7e0d954bcfbea528707ef0ea285382fbeb982"
+# Installed source and model identity reviewed against the locked packages.
+_SDK_FILES: dict[str, tuple[str, dict[str, tuple[int, str]]]] = {
+ "boto3": (
+ "1.43.102",
+ {
+ "boto3/session.py": (
+ 22106,
+ "c113623eabf46a1438af30940513cdd54b94ede6d7b6894743d5f7e1d18b9a78",
+ ),
+ "boto3-1.43.102.dist-info/METADATA": (
+ 6571,
+ "ecfda0e6c61eba7a5813b0be14364808ad56ad66e9af31b9a6a07e9a5d27fccb",
+ ),
+ },
+ ),
+ "botocore": (
+ "1.43.102",
+ {
+ "botocore/endpoint.py": (
+ 17469,
+ "0ce924a118e3ad98a2c71293541508efc1b3e0f84fb1b80272394829ba656262",
+ ),
+ "botocore/httpsession.py": (
+ 20785,
+ "14b2961314c711499da785ec30f00d9b04ead55a4403b2aba5c5e822ffd93b7b",
+ ),
+ "botocore/awsrequest.py": (
+ 23375,
+ "85d0ae028b154a991db1078c70987440c290c55b9c34f66e8ae4053de5d22c2f",
+ ),
+ "botocore/response.py": (
+ 7865,
+ "e2ee6f81ec35f5dae9879f5f5c600dca22b78a2c43cefb0dac3b8b894ce9d11d",
+ ),
+ "botocore/parsers.py": (
+ 60629,
+ "2367dc4b9f07e495185c65ea6e9d37014133968de19f860822f48b1a7e1f26ea",
+ ),
+ "botocore/client.py": (
+ 54779,
+ "0e7b790ccb6aa2ca34803c6878aaead7db9108eee4e3fba0aa66008b647967f9",
+ ),
+ "botocore/hooks.py": (
+ 25053,
+ "a484c295983cf1e8905f73ec435ff7db0c9d8b9d8b24321a9c5d18bdfd81e312",
+ ),
+ "botocore/config.py": (
+ 20566,
+ "3ff88b9f5af4b19a41792e5696918135e0bfc64011dcb4843fb515922239014f",
+ ),
+ "botocore/utils.py": (
+ 138508,
+ "e9d9182eacf6881499284a357fa1759796d7f57bc32930b691a92ad9915f2530",
+ ),
+ "botocore/data/s3/2006-03-01/service-2.json.gz": (
+ 184637,
+ "e338f63ec7e05cc17ba0bb77ab9a2608ddff1bcba490f672ecc2c9bc32761c33",
+ ),
+ "botocore-1.43.102.dist-info/METADATA": (
+ 5570,
+ "fe68d4befe049f7c6cd63df10c32f6d1a629a5561d1877d655323c5bcb9886c7",
+ ),
+ "botocore/handlers.py": (
+ 68835,
+ "504f9fd31a56a197dd63ee46796f86de471c02e550b2813a5fbcfacf3bcd6819",
+ ),
+ "botocore/retries/standard.py": (
+ 27817,
+ "ba5ede051c74c789ce59b03e9dff32c4da252190032ff7f512479b8ce1cc598b",
+ ),
+ "botocore/model.py": (
+ 32334,
+ "1f502c7ddf1ca329b695e30d36eff773fd2c483434ef1f1d9bc27582d1c38b8a",
+ ),
+ "botocore/signers.py": (
+ 34984,
+ "3340dfd41a217eea8a8418eabeab1a00c68a21057b75c3b094c0fc33d2e1f17e",
+ ),
+ "botocore/loaders.py": (
+ 18826,
+ "aa2be08c3dd386f52681b27e325d28664cd9c6b9cdbd6cf8eff7d2eda66aea42",
+ ),
+ "botocore/data/s3/2006-03-01/service-2.sdk-extras.json": (
+ 98,
+ "1617a38a6d78af2b696fb4e16bfd18eb72ad73aa9d247bc078643cf9dfd66408",
+ ),
+ "botocore/data/s3/2006-03-01/endpoint-rule-set-1.json.gz": (
+ 23219,
+ "f0127007d896917ddc62361ac8db64f47be2c7470f6ae290afdca587cc2758cb",
+ ),
+ },
+ ),
+ "urllib3": (
+ "2.8.0",
+ {
+ "urllib3/response.py": (
+ 54628,
+ "0e4559c737914d4f8f941dca9ecfaf6caec6792258302bffec7b8b41dae3c71a",
+ ),
+ "urllib3-2.8.0.dist-info/METADATA": (
+ 7389,
+ "10898c620e8007c030e07fa5622b68358a43010025dfbd78a1cb797699de2bb4",
+ ),
+ },
+ ),
+}
+
+
+class ProbeError(object_store.ObjectStoreError):
+ """The bounded bucket proof is unavailable or incomplete."""
+
+
+def _refuse() -> ProbeError:
+ # Do not copy request URLs, keys, headers or response bodies into diagnostics.
+ return ProbeError("object store cell probe refused", operation="cell probe")
+
+
+def _regular_digest(path: Path, size: int) -> str:
+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+ with os.fdopen(fd, "rb") as stream:
+ info = os.fstat(stream.fileno())
+ if not stat.S_ISREG(info.st_mode) or info.st_size != size:
+ raise _refuse()
+ return hashlib.sha256(stream.read(size + 1)).hexdigest()
+
+
+def require_sdk() -> None:
+ """Refuse an SDK whose transport/parser/model contract was not reviewed."""
+ if os.environ.get("AWS_DATA_PATH"):
+ raise _refuse()
+ set_query_tag("cell-attach-bucket-probe")
+ try:
+ for name, (version, files) in _SDK_FILES.items():
+ dist = metadata.distribution(name)
+ if dist.version != version:
+ raise _refuse()
+ for relative, (size, digest) in files.items():
+ if _regular_digest(Path(cast(Path, dist.locate_file(relative))), size) != digest:
+ raise _refuse()
+ except (OSError, metadata.PackageNotFoundError, ValueError) as exc:
+ raise _refuse() from exc
+
+
+def _date(value: Any) -> datetime:
+ if not isinstance(value, str):
+ raise _refuse()
+ try:
+ at = parsedate_to_datetime(value)
+ if at.tzinfo is None or format_datetime(at, usegmt=True) != value:
+ raise _refuse()
+ return at.astimezone(UTC)
+ except (ValueError, TypeError, OverflowError) as exc:
+ raise _refuse() from exc
+
+
+class _Raw:
+ """Only bounded reads reach the signed client's original raw response."""
+
+ def __init__(self, raw: Any, cap: int, declared: Any) -> None:
+ self.raw, self.cap, self.seen = raw, cap, 0
+ self.closed = False
+ self.declared: int | None = None
+ try:
+ if declared is not None:
+ if not isinstance(declared, str) or not re.fullmatch(r"[0-9]{1,10}", declared):
+ raise _refuse()
+ self.declared = int(declared)
+ if self.declared > cap:
+ raise _refuse()
+ except BaseException:
+ self._close_failed()
+ raise
+
+ def _close_failed(self) -> None:
+ try:
+ self.close()
+ except Exception: # noqa: BLE001 — retain the original read or validation failure
+ pass
+
+ def close(self) -> None:
+ if not self.closed:
+ self.closed = True
+ self.raw.close()
+
+ def _next(self, amount: int) -> bytes:
+ try:
+ asked = min(amount, _CHUNK, self.cap - self.seen + 1)
+ chunk = self.raw.read(asked)
+ if not isinstance(chunk, bytes) or len(chunk) > asked:
+ raise _refuse()
+ self.seen += len(chunk)
+ if self.seen > self.cap or (self.declared is not None and self.seen > self.declared):
+ raise _refuse()
+ if not chunk:
+ if self.declared is not None and self.seen != self.declared:
+ raise _refuse()
+ self.close()
+ return chunk
+ except BaseException:
+ self._close_failed()
+ raise
+
+ def read(self, amt: int | None = None) -> bytes:
+ if amt == 0 or self.closed:
+ return b""
+ if amt is not None and (type(amt) is not int or amt < 0):
+ self._close_failed()
+ raise _refuse()
+ set_query_tag("cell-attach-bucket-probe")
+ chunks: list[bytes] = []
+ size = 0
+ while amt is None or size < amt:
+ chunk = self._next(self.cap + 1 if amt is None else amt - size)
+ if not chunk:
+ break
+ chunks.append(chunk)
+ size += len(chunk)
+ return b"".join(chunks)
+
+ def stream(self, amt: int = _CHUNK, decode_content: bool = False) -> Iterator[bytes]:
+ if type(amt) is not int or amt <= 0 or decode_content:
+ self._close_failed()
+ raise _refuse()
+ set_query_tag("cell-attach-bucket-probe")
+ try:
+ while chunk := self.read(min(amt, _CHUNK)):
+ yield chunk
+ finally:
+ self.close()
+
+
+def _send(client: Any, operation: str, request: Any, **_: Any) -> Any:
+ if request.context.get(_SENT):
+ raise _refuse()
+ request.context[_SENT] = True
+ previous = request.stream_output
+ try:
+ request.stream_output = True
+ response = client._endpoint.http_session.send(request)
+ finally:
+ request.stream_output = previous
+ raw = response.raw
+ try:
+ from botocore.awsrequest import AWSResponse
+
+ if (
+ type(response) is not AWSResponse
+ or type(response.status_code) is not int
+ or response._content is not None
+ ):
+ raise _refuse()
+ cap = _LIST_BYTES if operation == "ListObjectsV2" else _ERROR_BYTES
+ if operation == "GetObject" and response.status_code < 300:
+ cap = _MARKER_BYTES + 1
+ response.raw = _Raw(raw, cap, response.headers.get("content-length"))
+ _date(response.headers.get("date"))
+ if response.headers.get("content-encoding", "identity").lower() != "identity":
+ raise _refuse()
+ if response.status_code >= 300:
+ # Refuse before S3's region redirect handler can issue another call.
+ _ = response.content
+ raise _refuse()
+ if operation == "GetObject":
+ # Close the connection before parsing even malformed GET headers.
+ value = response.raw.read()
+ response.raw = _Raw(io.BytesIO(value), cap, str(len(value)))
+ return response
+ except BaseException:
+ try:
+ response.raw.close()
+ except Exception: # noqa: BLE001 — retain the original bounded transport failure
+ pass
+ raise
+
+
+def _no_retry(
+ *, response: Any, attempts: int, caught_exception: BaseException | None, **_: Any
+) -> None:
+ # Exact-operation events precede S3's wildcard region/HeadBucket handler.
+ if caught_exception is not None:
+ raise caught_exception
+ if (
+ attempts != 1
+ or response is None
+ or not isinstance(response[1], dict)
+ or "Error" in response[1]
+ ):
+ raise _refuse()
+
+
+def _ordinary_signing(*, request: Any, signing_name: str, **_: Any) -> None:
+ # Express auth would create a session through another S3 operation.
+ effective = request.context.get("signing", {}).get("signing_name", signing_name)
+ if effective != "s3":
+ raise _refuse()
+
+
+def install(client: Any) -> None:
+ """Install only these four operations on the fixed diagnostic client."""
+ from functools import partial
+
+ model = client.meta.service_model
+ description = json.dumps(model._service_description, sort_keys=True, separators=(",", ":"))
+ if (
+ model.service_name != "s3"
+ or hashlib.sha256(description.encode()).hexdigest() != _MODEL_SHA256
+ ):
+ raise _refuse()
+ set_query_tag("cell-attach-bucket-probe")
+ for operation in ("ListObjectsV2", "GetObject", "PutObject", "DeleteObject"):
+ client.meta.events.register_first(
+ f"before-sign.s3.{operation}",
+ _ordinary_signing,
+ unique_id=f"loom-cell-probe-signing-{operation}",
+ )
+ client.meta.events.register_first(
+ f"before-send.s3.{operation}",
+ partial(_send, client, operation),
+ unique_id=f"loom-cell-probe-{operation}",
+ )
+ client.meta.events.register_first(
+ f"needs-retry.s3.{operation}",
+ _no_retry,
+ unique_id=f"loom-cell-probe-no-retry-{operation}",
+ )
+
+
+def _client() -> Any:
+ return object_store._client_for(
+ settings.document_s3_endpoint_url,
+ settings.document_s3_region,
+ settings.document_s3_connect_timeout_s,
+ settings.document_s3_read_timeout_s,
+ diagnostic=True,
+ )
+
+
+def _ack(response: Any, status: int) -> datetime:
+ if not isinstance(response, dict):
+ raise _refuse()
+ meta = response.get("ResponseMetadata")
+ if not isinstance(meta, dict) or type(meta.get("HTTPStatusCode")) is not int:
+ raise _refuse()
+ headers = meta.get("HTTPHeaders")
+ if meta["HTTPStatusCode"] != status or not isinstance(headers, dict):
+ raise _refuse()
+ return _date(headers.get("date"))
+
+
+def _etag(value: Any) -> str:
+ if not isinstance(value, str) or value in ("*", '"*"') or not _ETAG.fullmatch(value):
+ raise _refuse()
+ return value
+
+
+def _location() -> object_store.Location:
+ name = settings.cell
+ if not settings.document_s3_bucket or not isinstance(name, str) or not NAME.fullmatch(name):
+ raise _refuse()
+ key = f"{object_store.root_prefix()}_cell_attach_probe/{name}"
+ if len(key.encode("utf-8")) > object_store.MAX_KEY:
+ raise _refuse()
+ return object_store.Location(settings.document_s3_bucket, key)
+
+
+def _value(value: bytes) -> None:
+ if not isinstance(value, bytes) or not 1 <= len(value) <= _MARKER_BYTES:
+ raise _refuse()
+
+
+@dataclass(frozen=True)
+class Marker:
+ value: bytes
+ etag: str
+ at: datetime
+
+
+def _read(client: Any, loc: object_store.Location) -> Marker:
+ response = client.get_object(Bucket=loc.bucket, Key=loc.key)
+ body = response.get("Body")
+ if body is None:
+ raise _refuse()
+ try:
+ at = _ack(response, 200)
+ etag = _etag(response.get("ETag"))
+ value = body.read()
+ _value(value)
+ marker = Marker(value, etag, at)
+ except BaseException:
+ try:
+ body.close()
+ except Exception: # noqa: BLE001 — retain the original marker read failure
+ pass
+ raise
+ else:
+ body.close()
+ return marker
+
+
+async def read_marker() -> Marker:
+ loc = _location()
+ async with object_store.transfer() as lease:
+ return await object_store._thread(lambda: _read(_client(), loc), lease)
+
+
+async def put_marker(value: bytes) -> datetime:
+ _value(value)
+ loc = _location()
+
+ def put() -> datetime:
+ response = _client().put_object(Bucket=loc.bucket, Key=loc.key, Body=value)
+ at = _ack(response, 200)
+ _etag(response.get("ETag"))
+ return at
+
+ async with object_store.transfer() as lease:
+ return await object_store._thread(put, lease)
+
+
+async def delete_marker(expected: bytes) -> None:
+ """Delete only the exact bytes read with the ETag sent in IfMatch."""
+ _value(expected)
+ loc = _location()
+
+ def delete() -> None:
+ client = _client()
+ marker = _read(client, loc)
+ if marker.value != expected:
+ raise _refuse()
+ response = client.delete_object(Bucket=loc.bucket, Key=loc.key, IfMatch=marker.etag)
+ _ack(response, 204)
+ if response.get("DeleteMarker", False) is not False:
+ raise _refuse()
+
+ async with object_store.transfer() as lease:
+ await object_store._thread(delete, lease)
+
+
+def _page(response: Any, prefix: str, after: str, maximum: int) -> object_store.Page:
+ at = _ack(response, 200)
+ if (
+ response.get("Name") != settings.document_s3_bucket
+ or response.get("Prefix") != prefix
+ or type(response.get("MaxKeys")) is not int
+ or response["MaxKeys"] != maximum
+ or type(response.get("IsTruncated")) is not bool
+ or type(response.get("KeyCount")) is not int
+ or (after and response.get("StartAfter") != after)
+ ):
+ raise _refuse()
+ entries = response.get("Contents", [])
+ if (
+ not isinstance(entries, list)
+ or len(entries) > maximum
+ or response["KeyCount"] != len(entries)
+ ):
+ raise _refuse()
+ listed: list[object_store.Listed] = []
+ previous = after
+ set_query_tag("cell-attach-bucket-probe")
+ for entry in entries:
+ if not isinstance(entry, dict):
+ raise _refuse()
+ key, modified = entry.get("Key"), entry.get("LastModified")
+ if (
+ not isinstance(key, str)
+ or not key.startswith(prefix)
+ or key <= previous
+ or not 1 <= len(key.encode("utf-8")) <= 1024
+ or not isinstance(modified, datetime)
+ or modified.tzinfo is None
+ ):
+ raise _refuse()
+ listed.append(object_store.Listed(key, modified.astimezone(UTC)))
+ previous = key
+ if response["IsTruncated"] and not listed:
+ raise _refuse()
+ return object_store.Page(listed, response["IsTruncated"], at)
+
+
+async def list_page(*, start_after: str = "", max_keys: int = 1000) -> object_store.Page:
+ """One strictly ordered page of this deployment's configured root only."""
+ _location()
+ prefix = object_store.root_prefix()
+ if (
+ type(max_keys) is not int
+ or not 1 <= max_keys <= 1000
+ or not isinstance(start_after, str)
+ or len(start_after.encode("utf-8")) > 1024
+ or (start_after and not start_after.startswith(prefix))
+ ):
+ raise _refuse()
+ args = {"Bucket": settings.document_s3_bucket, "Prefix": prefix, "MaxKeys": max_keys}
+ if start_after:
+ args["StartAfter"] = start_after
+ slots = object_store._semaphore(object_store._LISTINGS, object_store.LISTING_SLOTS)
+ await slots.acquire()
+ lease = object_store.Lease(slots)
+ try:
+ return await object_store._thread(
+ lambda: _page(_client().list_objects_v2(**args), prefix, start_after, max_keys), lease
+ )
+ finally:
+ lease.close()
+
+
+async def check_prefix() -> None:
+ """Exhaust the configured root and refuse another cell's fixed marker."""
+ own = _location()
+ name = own.key.rsplit("/", 1)[1]
+ after = ""
+ objects = key_bytes = 0
+ set_query_tag("cell-attach-bucket-probe")
+ for _ in range(100):
+ page = await list_page(start_after=after, max_keys=1000)
+ previous = after
+ for item in page.objects:
+ size = len(item.key.encode("utf-8"))
+ if item.key <= after or objects >= 100000 or key_bytes + size > 16 * 1024 * 1024:
+ raise _refuse()
+ objects += 1
+ key_bytes += size
+ after = item.key
+ _, reserved, holder = item.key.rpartition("/_cell_attach_probe/")
+ if reserved and NAME.fullmatch(holder) and holder != name:
+ raise _refuse()
+ if not page.more:
+ return
+ if after == previous or objects == 100000:
+ raise _refuse()
+ raise _refuse()
diff --git a/tests/unit/test_object_store_probe.py b/tests/unit/test_object_store_probe.py
new file mode 100644
index 0000000000..9e709a4d48
--- /dev/null
+++ b/tests/unit/test_object_store_probe.py
@@ -0,0 +1,733 @@
+"""Fixed bucket diagnostics preserve ordinary clients and bound SDK parsing."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from server.core import object_store
+from server.core import object_store_probe as probe
+
+DATE = "Wed, 18 Nov 2026 12:00:00 GMT"
+AT = datetime(2026, 11, 18, 12, tzinfo=UTC)
+
+
+class Raw:
+ def __init__(self, value: bytes, *, fail: BaseException | None = None) -> None:
+ self.body = io.BytesIO(value)
+ self.fail = fail
+ self.reads: list[int] = []
+ self.closed = False
+
+ def read(self, amount: int = -1) -> bytes:
+ self.reads.append(amount)
+ if self.fail is not None:
+ raise self.fail
+ return self.body.read(amount)
+
+ def close(self) -> None:
+ self.closed = True
+ self.body.close()
+
+
+def ack(status: int = 200, **fields: Any) -> dict[str, Any]:
+ return {"ResponseMetadata": {"HTTPStatusCode": status, "HTTPHeaders": {"date": DATE}}, **fields}
+
+
+def test_raw_exact_limit_and_cap_plus_one_close_before_join() -> None:
+ raw = Raw(b"x" * probe._LIST_BYTES)
+ assert (
+ len(probe._Raw(raw, probe._LIST_BYTES, str(probe._LIST_BYTES)).read()) == probe._LIST_BYTES
+ )
+ assert raw.closed and max(raw.reads) <= 64 * 1024
+ overflow = Raw(b"x" * (probe._LIST_BYTES + 1))
+ with pytest.raises(probe.ProbeError):
+ b"".join(probe._Raw(overflow, probe._LIST_BYTES, None).stream())
+ assert overflow.closed and sum(overflow.reads) == probe._LIST_BYTES + 1
+
+
+@pytest.mark.parametrize("length", ["65537", "-1", "1x", True, 65536])
+def test_declared_error_length_refuses_before_a_read(length: Any) -> None:
+ raw = Raw(b"payload")
+ with pytest.raises(probe.ProbeError):
+ probe._Raw(raw, 65536, length)
+ assert raw.closed and not raw.reads
+
+
+@pytest.mark.parametrize("value,declared", [(b"abc", "2"), (b"a", "2")])
+def test_actual_length_mismatch_closes(value: bytes, declared: str) -> None:
+ raw = Raw(value)
+ with pytest.raises(probe.ProbeError):
+ probe._Raw(raw, 65536, declared).read()
+ assert raw.closed
+
+
+def test_raw_preserves_the_original_interruption() -> None:
+ original = KeyboardInterrupt()
+ raw = Raw(b"", fail=original)
+ with pytest.raises(KeyboardInterrupt) as raised:
+ probe._Raw(raw, 1025, None).read()
+ assert raised.value is original and raw.closed
+
+
+@pytest.mark.parametrize("value", [None, "", "not a date", "Wed, 18 Nov 2026 12:00:00", True])
+def test_no_local_clock_date_fallback(value: Any) -> None:
+ with pytest.raises(probe.ProbeError):
+ probe._date(value)
+ assert probe._date(DATE) == AT
+
+
+@pytest.fixture
+def configured(settings_override: Any) -> None:
+ settings_override(cell="west", document_s3_bucket="bucket", document_s3_prefix="root")
+
+
+def page(*, keys: tuple[str, ...] = ("root/a", "root/b"), more: bool = False) -> dict[str, Any]:
+ return ack(
+ Name="bucket",
+ Prefix="root/",
+ MaxKeys=2,
+ KeyCount=len(keys),
+ IsTruncated=more,
+ Contents=[{"Key": key, "LastModified": AT} for key in keys],
+ )
+
+
+def test_strict_page_is_the_actual_object_store_page(configured: None) -> None:
+ result = probe._page(page(more=True), "root/", "", 2)
+ assert type(result) is object_store.Page and result.more and result.at == AT
+ assert result.objects == [object_store.Listed("root/a", AT), object_store.Listed("root/b", AT)]
+
+
+@pytest.mark.parametrize(
+ "mutation",
+ [
+ {"IsTruncated": 1},
+ {"KeyCount": True},
+ {"KeyCount": 3},
+ {"MaxKeys": True},
+ {"Name": "other"},
+ {"Prefix": "other/"},
+ {"Contents": None},
+ {"Contents": [{"Key": "root/a", "LastModified": datetime(2026, 1, 1)}], "KeyCount": 1},
+ {"IsTruncated": True, "Contents": [], "KeyCount": 0},
+ ],
+)
+def test_partial_or_malformed_page_refuses(configured: None, mutation: dict[str, Any]) -> None:
+ response = page() | mutation
+ with pytest.raises(probe.ProbeError):
+ probe._page(response, "root/", "", 2)
+
+
+@pytest.mark.parametrize(
+ "keys",
+ [
+ ("root/b", "root/a"),
+ ("root/a", "root/a"),
+ ("foreign/a",),
+ ("root/" + "x" * 1020,),
+ ("root/a", "root/b", "root/c"),
+ ],
+)
+def test_cursor_order_foreign_key_and_size_refuse(configured: None, keys: tuple[str, ...]) -> None:
+ with pytest.raises(probe.ProbeError):
+ probe._page(page(keys=keys), "root/", "", 2)
+
+
+def test_page_requires_exact_resume_echo_and_strict_progress(configured: None) -> None:
+ response = page()
+ with pytest.raises(probe.ProbeError):
+ probe._page(response, "root/", "root/a", 2)
+ response["StartAfter"] = "root/a"
+ with pytest.raises(probe.ProbeError):
+ probe._page(response, "root/", "root/a", 2)
+ response.update(KeyCount=1, Contents=[{"Key": "root/b", "LastModified": AT}])
+ assert probe._page(response, "root/", "root/a", 2).objects == [
+ object_store.Listed("root/b", AT)
+ ]
+
+
+def test_only_the_configured_own_marker_can_be_addressed(
+ configured: None, settings_override: Any
+) -> None:
+ assert probe._location() == object_store.Location("bucket", "root/_cell_attach_probe/west")
+ settings_override(cell="other/path")
+ with pytest.raises(probe.ProbeError):
+ probe._location()
+ settings_override(cell="west", document_s3_prefix="x" * 510)
+ with pytest.raises(probe.ProbeError):
+ probe._location()
+ settings_override(document_s3_bucket="")
+ with pytest.raises(probe.ProbeError):
+ probe._location()
+
+
+@pytest.mark.parametrize("etag", ["*", '"*"', "", "bare", 'W/"weak"', '"bad\nvalue"', None])
+def test_conditional_delete_requires_an_exact_strong_etag(etag: Any) -> None:
+ with pytest.raises(probe.ProbeError):
+ probe._etag(etag)
+
+
+class MarkerClient:
+ def __init__(self, value: bytes = b"own", etag: str = '"first"') -> None:
+ self.body = Raw(value)
+ self.etag = etag
+ self.deletes: list[dict[str, Any]] = []
+ self.ack = ack(204)
+
+ def get_object(self, **kw: Any) -> dict[str, Any]:
+ assert kw == {"Bucket": "bucket", "Key": "root/_cell_attach_probe/west"}
+ # The actual transport bound is exercised separately through botocore.
+ return ack(ETag=self.etag, Body=self.body)
+
+ def delete_object(self, **kw: Any) -> dict[str, Any]:
+ self.deletes.append(kw)
+ return self.ack
+
+
+def test_read_returns_bytes_and_etag_from_the_same_closed_get(configured: None) -> None:
+ client = MarkerClient()
+ assert probe._read(client, probe._location()) == probe.Marker(b"own", '"first"', AT)
+ assert client.body.closed
+
+
+@pytest.mark.asyncio
+async def test_delete_uses_exact_owned_bytes_and_ifmatch(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ client = MarkerClient()
+ monkeypatch.setattr(probe, "_client", lambda: client)
+ await probe.delete_marker(b"own")
+ assert client.body.closed
+ assert client.deletes == [
+ {"Bucket": "bucket", "Key": "root/_cell_attach_probe/west", "IfMatch": '"first"'}
+ ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "value,etag", [(b"other", '"first"'), (b"own", "*"), (b"x" * 1025, '"first"')]
+)
+async def test_foreign_or_invalid_marker_never_deletes(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ value: bytes,
+ etag: str,
+) -> None:
+ client = MarkerClient(value, etag)
+ monkeypatch.setattr(probe, "_client", lambda: client)
+ with pytest.raises(probe.ProbeError):
+ await probe.delete_marker(b"own")
+ assert client.body.closed and not client.deletes
+
+
+@pytest.mark.asyncio
+async def test_unsupported_delete_is_incomplete_without_an_unconditional_retry(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ client = MarkerClient()
+ client.ack = ack(200)
+ monkeypatch.setattr(probe, "_client", lambda: client)
+ with pytest.raises(probe.ProbeError):
+ await probe.delete_marker(b"own")
+ assert len(client.deletes) == 1 and client.deletes[0]["IfMatch"] == '"first"'
+
+
+@pytest.fixture
+def sdk_client(monkeypatch: pytest.MonkeyPatch) -> Any:
+ # Synthetic unit credentials only; all transport is replaced on this client.
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "unit-access")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "unit-secret")
+ monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
+ monkeypatch.delenv("AWS_PROFILE", raising=False)
+ monkeypatch.delenv("AWS_CONFIG_FILE", raising=False)
+ monkeypatch.delenv("AWS_DATA_PATH", raising=False)
+ object_store._client_for.cache_clear()
+ client = object_store._client_for("https://store.invalid", "us-east-1", 1, 2, diagnostic=True)
+ try:
+ yield client
+ finally:
+ client.close()
+ object_store._client_for.cache_clear()
+
+
+def response(status: int, value: bytes, *, date: str = DATE) -> tuple[Any, Raw]:
+ from botocore.awsrequest import AWSResponse
+
+ raw = Raw(value)
+ return AWSResponse(
+ "https://store.invalid", status, {"date": date, "content-length": str(len(value))}, raw
+ ), raw
+
+
+@pytest.mark.parametrize("status", [200, 307])
+def test_actual_sdk_error_seam_sends_once_without_headbucket(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+ status: int,
+) -> None:
+ value = b"PermanentRedirectelsewhere.invalid"
+ reply, raw = response(status, value)
+ sent: list[Any] = []
+
+ def send(request: Any) -> Any:
+ sent.append(request)
+ assert request.stream_output is True
+ return reply
+
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", send)
+ with pytest.raises(probe.ProbeError):
+ sdk_client.list_objects_v2(Bucket="bucket", Prefix="root/", MaxKeys=2)
+ assert len(sent) == 1 and sent[0].stream_output is False and raw.closed
+
+
+def test_actual_sdk_list_uses_ordinary_parser_after_raw_cap(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+ configured: None,
+) -> None:
+ value = b'bucketroot/21falseroot/a2026-11-18T12:00:00.000Z"e"1'
+ reply, raw = response(200, value)
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", lambda request: reply)
+ parsed = sdk_client.list_objects_v2(Bucket="bucket", Prefix="root/", MaxKeys=2)
+ result = probe._page(parsed, "root/", "", 2)
+ assert result.objects == [object_store.Listed("root/a", AT)] and not result.more
+ assert raw.closed and max(raw.reads) <= 64 * 1024
+
+
+def test_actual_sdk_get_cannot_materialize_marker_overflow(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+ configured: None,
+) -> None:
+ reply, raw = response(200, b"x" * 1026)
+ reply.headers["etag"] = '"e"'
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", lambda request: reply)
+ with pytest.raises(probe.ProbeError):
+ probe._read(sdk_client, probe._location())
+ assert raw.closed and not raw.reads
+
+
+def test_send_restores_request_flag_on_transport_failure() -> None:
+ original = KeyboardInterrupt()
+
+ def send(request: Any) -> Any:
+ assert request.stream_output is True
+ raise original
+
+ request = SimpleNamespace(context={}, stream_output=False)
+ client = SimpleNamespace(_endpoint=SimpleNamespace(http_session=SimpleNamespace(send=send)))
+ with pytest.raises(KeyboardInterrupt) as raised:
+ probe._send(client, "PutObject", request)
+ assert raised.value is original and request.stream_output is False
+ with pytest.raises(probe.ProbeError):
+ probe._send(client, "PutObject", request)
+
+
+def test_shared_factory_preserves_ordinary_client_policy(sdk_client: Any) -> None:
+ ordinary = object_store._client_config(2.5, 45)
+ diagnostic = object_store._client_config(2.5, 45, diagnostic=True)
+ assert ordinary.retries == {"max_attempts": 3, "mode": "standard"}
+ assert diagnostic.retries == {"total_max_attempts": 1, "mode": "standard"}
+ ordinary_options = dict(ordinary._user_provided_options)
+ diagnostic_options = dict(diagnostic._user_provided_options)
+ ordinary_options.pop("retries")
+ diagnostic_options.pop("retries")
+ assert ordinary_options == diagnostic_options
+ assert diagnostic.max_pool_connections == object_store._THREADS == 16
+ assert sdk_client is object_store._client_for(
+ "https://store.invalid", "us-east-1", 1, 2, diagnostic=True
+ )
+
+
+def test_sdk_source_identity_refuses_a_changed_installed_file(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(probe, "_regular_digest", lambda path, size: "0" * 64)
+ with pytest.raises(probe.ProbeError):
+ probe.require_sdk()
+
+
+def test_actual_sdk_get_closes_transport_before_parser_and_keeps_marker_bytes(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+ configured: None,
+) -> None:
+ reply, raw = response(200, b"own")
+ reply.headers["etag"] = '"first"'
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", lambda request: reply)
+ assert probe._read(sdk_client, probe._location()) == probe.Marker(b"own", '"first"', AT)
+ assert raw.closed and max(raw.reads) <= 1025 + 1
+
+
+def test_actual_sdk_error_bytes_are_capped_before_parsing(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ reply, raw = response(403, b"x" * (65536 + 1))
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", lambda request: reply)
+ with pytest.raises(probe.ProbeError):
+ sdk_client.put_object(Bucket="bucket", Key="root/_cell_attach_probe/west", Body=b"own")
+ assert raw.closed and not raw.reads
+
+
+def test_actual_sdk_needs_retry_guard_precedes_wildcard_region_handler(
+ sdk_client: Any,
+) -> None:
+ later: list[bool] = []
+ sdk_client.meta.events.register_first("needs-retry.s3", lambda **kwargs: later.append(True))
+ with pytest.raises(probe.ProbeError):
+ sdk_client.meta.events.emit(
+ "needs-retry.s3.ListObjectsV2",
+ response=(SimpleNamespace(status_code=200), {"Error": {"Code": "PermanentRedirect"}}),
+ attempts=1,
+ caught_exception=None,
+ )
+ assert not later
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("status", [412, 501])
+async def test_actual_sdk_replacement_or_unsupported_delete_leaves_slot_incomplete(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+ configured: None,
+ status: int,
+) -> None:
+ requests: list[Any] = []
+ raws: list[Raw] = []
+
+ def send(request: Any) -> Any:
+ requests.append(request)
+ if request.method == "GET":
+ reply, raw = response(200, b"own")
+ reply.headers["etag"] = '"first"'
+ else:
+ assert request.method == "DELETE" and request.headers["If-Match"] == b'"first"'
+ reply, raw = response(status, b"PreconditionFailed")
+ raws.append(raw)
+ return reply
+
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", send)
+ monkeypatch.setattr(probe, "_client", lambda: sdk_client)
+ with pytest.raises(probe.ProbeError):
+ await probe.delete_marker(b"own")
+ assert [r.method for r in requests] == ["GET", "DELETE"]
+ assert all(raw.closed for raw in raws)
+
+
+def test_actual_sdk_missing_date_closes_before_parser(
+ sdk_client: Any,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ reply, raw = response(200, b"", date="")
+ monkeypatch.setattr(sdk_client._endpoint.http_session, "send", lambda request: reply)
+ with pytest.raises(probe.ProbeError):
+ sdk_client.put_object(Bucket="bucket", Key="root/_cell_attach_probe/west", Body=b"own")
+ assert raw.closed and not raw.reads
+
+
+def test_ordinary_client_does_not_receive_diagnostic_hooks(sdk_client: Any) -> None:
+ from functools import partial
+
+ ordinary = object_store._client_for("https://store.invalid", "us-east-1", 1, 2)
+ try:
+ listeners = ordinary.meta.events._emitter._handlers.prefix_search(
+ "before-send.s3.ListObjectsV2"
+ )
+ assert ordinary is not sdk_client
+ assert ordinary.meta.config.retries == {"total_max_attempts": 4, "mode": "standard"}
+ assert not any(
+ isinstance(handler, partial) and handler.func is probe._send for handler in listeners
+ )
+ assert ordinary._endpoint.http_session is not sdk_client._endpoint.http_session
+ finally:
+ ordinary.close()
+
+
+def test_unsupported_sdk_version_refuses_before_transport(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ probe.metadata, "distribution", lambda name: SimpleNamespace(version="unsupported")
+ )
+ with pytest.raises(probe.ProbeError):
+ probe.require_sdk()
+
+
+def test_actual_sdk_express_signing_refuses_before_session_resolver(sdk_client: Any) -> None:
+ later: list[bool] = []
+ sdk_client.meta.events.register_first("before-sign.s3", lambda **kwargs: later.append(True))
+ request = SimpleNamespace(context={"signing": {"signing_name": "s3express"}})
+ with pytest.raises(probe.ProbeError):
+ sdk_client.meta.events.emit(
+ "before-sign.s3.GetObject",
+ request=request,
+ signing_name="s3",
+ signature_version="v4-s3express",
+ )
+ assert not later
+
+
+def test_custom_sdk_model_path_is_unsupported_without_importing_it(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("AWS_DATA_PATH", "/not-opened/unit-model")
+ with pytest.raises(probe.ProbeError):
+ probe.require_sdk()
+
+
+def test_a_changed_loaded_service_model_is_not_installed(
+ sdk_client: Any, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(probe, "_MODEL_SHA256", "0" * 64)
+ with pytest.raises(probe.ProbeError):
+ probe.install(sdk_client)
+
+
+def prefix_reader(monkeypatch: pytest.MonkeyPatch, make: Any) -> list[str]:
+ """Project synthetic wire pages through the real strict page validator."""
+ calls: list[str] = []
+
+ async def read(*, start_after: str = "", max_keys: int = 1000) -> object_store.Page:
+ assert max_keys == 1000
+ index = len(calls)
+ calls.append(start_after)
+ response = make(index, start_after)
+ return probe._page(response, "root/", start_after, max_keys)
+
+ monkeypatch.setattr(probe, "list_page", read)
+ return calls
+
+
+def prefix_page(keys: tuple[str, ...], after: str, *, more: bool = False) -> dict[str, Any]:
+ return page(keys=keys, more=more) | {"MaxKeys": 1000, "StartAfter": after}
+
+
+@pytest.mark.asyncio
+async def test_prefix_exhausts_all_pages_and_exports_no_keys(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ keys = (("root/a", "root/b"), ("root/c",), ("root/z",))
+ calls = prefix_reader(
+ monkeypatch,
+ lambda index, after: prefix_page(keys[index], after, more=index < 2),
+ )
+ assert await probe.check_prefix() is None
+ assert calls == ["", "root/b", "root/c"]
+
+
+@pytest.mark.asyncio
+async def test_prefix_empty_final_page_finishes_without_an_extra_read(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls = prefix_reader(monkeypatch, lambda index, after: prefix_page((), after))
+ assert await probe.check_prefix() is None
+ assert calls == [""]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "key",
+ [
+ "root/_cell_attach_probe/east",
+ "root/nested/_cell_attach_probe/east",
+ "root/nested/more/_cell_attach_probe/east",
+ ],
+)
+async def test_prefix_foreign_fixed_marker_refuses_at_any_depth_without_export(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ key: str,
+) -> None:
+ calls = prefix_reader(
+ monkeypatch,
+ lambda index, after: prefix_page((key,), after, more=True),
+ )
+ with pytest.raises(probe.ProbeError) as raised:
+ await probe.check_prefix()
+ assert calls == [""] and key not in str(raised.value)
+
+
+@pytest.mark.asyncio
+async def test_prefix_own_marker_and_nonmarker_suffixes_are_not_foreign_markers(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ keys = tuple(
+ sorted(
+ (
+ "root/_cell_attach_probe/west",
+ "root/deep/_cell_attach_probe/west",
+ "root/deep/_cell_attach_probe/east/child",
+ "root/deep/_cell_attach_probe/east_suffix",
+ "root/deep/_cell_attach_probe/EAST",
+ "root/deep/not_cell_attach_probe/east",
+ )
+ )
+ )
+ calls = prefix_reader(monkeypatch, lambda index, after: prefix_page(keys, after))
+ assert await probe.check_prefix() is None
+ assert calls == [""]
+
+
+@pytest.mark.asyncio
+async def test_prefix_finds_a_foreign_marker_after_an_earlier_clean_page(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ keys = (("root/a",), ("root/z/_cell_attach_probe/east",))
+ calls = prefix_reader(
+ monkeypatch,
+ lambda index, after: prefix_page(keys[index], after, more=index == 0),
+ )
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ assert calls == ["", "root/a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("more", [False, True])
+async def test_prefix_exact_object_limit_never_fetches_page101(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ more: bool,
+) -> None:
+ def make(index: int, after: str) -> dict[str, Any]:
+ assert index < 100
+ keys = tuple(f"root/{number:06d}" for number in range(index * 1000, (index + 1) * 1000))
+ return prefix_page(keys, after, more=index < 99 or more)
+
+ calls = prefix_reader(monkeypatch, make)
+ if more:
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ else:
+ assert await probe.check_prefix() is None
+ assert len(calls) == 100 and calls[-1] == "root/098999"
+
+
+@pytest.mark.asyncio
+async def test_prefix_page_limit_refuses_even_when_object_count_is_lower(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls = prefix_reader(
+ monkeypatch,
+ lambda index, after: prefix_page((f"root/{index:06d}",), after, more=True),
+ )
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ assert len(calls) == 100
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("extra", [0, 1])
+async def test_prefix_utf8_bytes_exact_limit_and_limit_plus_one(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ extra: int,
+) -> None:
+ # 16383 * 1024 + 512 + 512 == 16 MiB; characters are mostly two-byte UTF-8.
+ def make(index: int, after: str) -> dict[str, Any]:
+ keys: list[str] = []
+ for number in range(index * 1000, min((index + 1) * 1000, 16385)):
+ length = 1024 if number < 16383 else 512 + (extra if number == 16384 else 0)
+ head = f"root/{number:06d}/"
+ remaining = length - len(head)
+ keys.append(head + "é" * (remaining // 2) + "x" * (remaining % 2))
+ return prefix_page(tuple(keys), after, more=index < 16)
+
+ calls = prefix_reader(monkeypatch, make)
+ if extra:
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ else:
+ assert await probe.check_prefix() is None
+ assert len(calls) == 17
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("second", ["root/a", "root/0"])
+async def test_prefix_duplicate_or_regressing_cross_page_key_refuses(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ second: str,
+) -> None:
+ calls = prefix_reader(
+ monkeypatch,
+ lambda index, after: prefix_page(
+ ("root/a" if index == 0 else second,), after, more=index == 0
+ ),
+ )
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ assert calls == ["", "root/a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "mutation",
+ [
+ {"IsTruncated": None},
+ {"IsTruncated": 1},
+ {"KeyCount": 2},
+ {"Contents": [], "KeyCount": 0, "IsTruncated": True},
+ {"StartAfter": "root/wrong"},
+ {"ResponseMetadata": {}},
+ ],
+)
+async def test_prefix_partial_page_never_becomes_exhaustion(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ mutation: dict[str, Any],
+) -> None:
+ def make(index: int, after: str) -> dict[str, Any]:
+ if index == 0:
+ return prefix_page(("root/a",), after, more=True)
+ return prefix_page(("root/b",), after) | mutation
+
+ calls = prefix_reader(monkeypatch, make)
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ assert calls == ["", "root/a"]
+
+
+@pytest.mark.asyncio
+async def test_prefix_page1001st_object_is_rejected_by_the_existing_page_owner(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ keys = tuple(f"root/{number:06d}" for number in range(1001))
+ calls = prefix_reader(monkeypatch, lambda index, after: prefix_page(keys, after))
+ with pytest.raises(probe.ProbeError):
+ await probe.check_prefix()
+ assert calls == [""]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("original", [RuntimeError("read failed"), asyncio.CancelledError()])
+async def test_prefix_keeps_the_original_read_failure_or_cancellation(
+ configured: None,
+ monkeypatch: pytest.MonkeyPatch,
+ original: BaseException,
+) -> None:
+ calls: list[str] = []
+
+ async def read(*, start_after: str = "", max_keys: int = 1000) -> object_store.Page:
+ calls.append(start_after)
+ if not start_after:
+ return probe._page(prefix_page(("root/a",), "", more=True), "root/", "", max_keys)
+ raise original
+
+ monkeypatch.setattr(probe, "list_page", read)
+ with pytest.raises(type(original)) as raised:
+ await probe.check_prefix()
+ assert raised.value is original and calls == ["", "root/a"]