diff --git a/.gitignore b/.gitignore
index 3ee158df..26b35ac2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,7 @@ db/data/voa/ctsop*_lsoa_2026/CTSOP_*.zip
db/data/tpa/lvt_model_2026/config.py
db/data/tpa/lvt_model_2026/cells.json
db/data/tpa/lvt_model_2026/LICENSE
+# UCL's house price per square metre deposit (ReShare 856542, chronicle#409) is two
+# zips of 3.2 and 4.4 GB, above GitHub's file limit and carrying Royal Mail address
+# data licensed for non-commercial use. R2 holds them; the manifest pins the bytes.
+db/data/ucl/house_price_per_square_metre_856542/856542_bundle_*.zip
diff --git a/changelog.d/r2-large-raw-uploads-409.changed.md b/changelog.d/r2-large-raw-uploads-409.changed.md
new file mode 100644
index 00000000..fe2eb1ca
--- /dev/null
+++ b/changelog.d/r2-large-raw-uploads-409.changed.md
@@ -0,0 +1 @@
+`publish-raw`, `fetch-artifact` and `publish-derived` upload files above Wrangler's 300 MiB limit to R2 through its S3-compatible API (chronicle#409). Each file goes up as a multipart upload in 100 MiB parts, with one part in memory at a time, and a failed part is retried on its own. Every file is hashed a chunk at a time instead of being read whole, and `fetch-artifact` streams downloads to disk.
diff --git a/changelog.d/ucl-house-price-per-square-metre-856542.added.md b/changelog.d/ucl-house-price-per-square-metre-856542.added.md
new file mode 100644
index 00000000..e9491077
--- /dev/null
+++ b/changelog.d/ucl-house-price-per-square-metre-856542.added.md
@@ -0,0 +1 @@
+Register UCL's House Price Per Square Metre in England and Wales, 1995-2023 (UK Data Service ReShare 856542), both bundles, raw-only in R2 (chronicle#408, item 6).
diff --git a/chronicle/artifacts.py b/chronicle/artifacts.py
index c3f10266..0ff134fa 100644
--- a/chronicle/artifacts.py
+++ b/chronicle/artifacts.py
@@ -15,12 +15,13 @@
import sqlite3
import subprocess
from threading import Thread
-from typing import Any, BinaryIO, Literal, get_args
+from typing import Any, BinaryIO, Iterator, Literal, get_args
from urllib.parse import unquote, urlparse
import httpx
import yaml
+from chronicle import r2_s3
from chronicle.epoch import EMIT_EPOCH, Epoch, canonicalize_key, hash_domain
from chronicle.yaml_io import safe_load as safe_load_yaml
@@ -29,6 +30,9 @@
DEFAULT_R2_DERIVED_BUCKET = "ledger-derived"
DEFAULT_R2_PREFIX = "raw"
DEFAULT_R2_DERIVED_PREFIX = "derived"
+#: ``wrangler r2 object put`` refuses objects above 300 MiB. Larger files go to R2
+#: through its S3-compatible API as multipart uploads (chronicle#409).
+WRANGLER_PUT_LIMIT_BYTES = 300 * 1024 * 1024
# New UK and New Zealand uploads are namespaced by country. US objects predate
# the country segment and deliberately keep their legacy ``raw/{source_id}``
@@ -53,6 +57,7 @@
"ons",
"scotgov",
"slc",
+ "ucl",
"voa",
"welshgov",
}
@@ -525,17 +530,13 @@ def fetch_source_artifact(
package_path=output,
)
fetched_at = datetime.now(UTC).replace(microsecond=0).isoformat()
- content, inferred_filename = _read_artifact(source_url)
- artifact_filename = filename or inferred_filename
+ artifact_filename = filename or _filename_from_url(source_url)
if not artifact_filename:
raise ValueError("Could not infer artifact filename; pass --filename.")
output.mkdir(parents=True, exist_ok=True)
local_path = output / artifact_filename
- local_path.write_bytes(content)
-
- sha256 = hashlib.sha256(content).hexdigest()
- size_bytes = len(content)
+ sha256, size_bytes = _read_artifact(source_url, local_path)
manifest_path = output / "manifest.yaml"
r2_location = ArtifactStorageLocation(
@@ -682,8 +683,7 @@ def publish_derived_artifacts(
relative_path = artifact_path.relative_to(input_path).as_posix()
if relative_path == "build_artifacts.jsonl":
continue
- content = artifact_path.read_bytes()
- sha256 = hashlib.sha256(content).hexdigest()
+ sha256, size_bytes = _hash_file(artifact_path)
location = ArtifactStorageLocation(
provider="r2",
bucket=r2_bucket,
@@ -708,7 +708,7 @@ def publish_derived_artifacts(
artifact_name=relative_path,
local_path=str(artifact_path),
sha256=sha256,
- size_bytes=len(content),
+ size_bytes=size_bytes,
r2_location=location,
upload=upload,
)
@@ -1016,6 +1016,12 @@ def _hash_stream(stream: BinaryIO) -> tuple[str, int]:
return digest.hexdigest(), size_bytes
+def _hash_file(path: Path) -> tuple[str, int]:
+ """Hash a file a chunk at a time: raw files can be several GB (chronicle#409)."""
+ with path.open("rb") as stream:
+ return _hash_stream(stream)
+
+
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
_MISSING_OBJECT_MARKER = re.compile(
r"(? str | None:
return None
-def _read_artifact(source_url: str) -> tuple[bytes, str]:
+def _read_artifact(source_url: str, local_path: Path) -> tuple[str, int]:
+ """Copy a source artifact to ``local_path`` a chunk at a time, hashing it.
+
+ The bytes go to a ``.partial`` file that takes the final name only once the
+ copy is complete, so a failed download leaves any earlier copy in place.
+ """
+ parsed = urlparse(source_url)
+ if parsed.scheme not in ("http", "https", "file", ""):
+ raise ValueError(f"Unsupported source URL scheme: {parsed.scheme}")
+ partial_path = local_path.with_name(f"{local_path.name}.partial")
+ digest = hashlib.sha256()
+ size_bytes = 0
+ try:
+ with partial_path.open("wb") as output:
+ for chunk in _artifact_chunks(source_url):
+ output.write(chunk)
+ digest.update(chunk)
+ size_bytes += len(chunk)
+ partial_path.replace(local_path)
+ except BaseException:
+ partial_path.unlink(missing_ok=True)
+ raise
+ return digest.hexdigest(), size_bytes
+
+
+def _artifact_chunks(source_url: str) -> Iterator[bytes]:
parsed = urlparse(source_url)
if parsed.scheme in ("http", "https"):
- response = httpx.get(source_url, follow_redirects=True, timeout=60)
- response.raise_for_status()
- return response.content, _filename_from_url(source_url)
- if parsed.scheme == "file":
- path = Path(unquote(parsed.path))
- return path.read_bytes(), path.name
- if not parsed.scheme:
- path = Path(source_url)
- return path.read_bytes(), path.name
- raise ValueError(f"Unsupported source URL scheme: {parsed.scheme}")
+ with httpx.stream(
+ "GET", source_url, follow_redirects=True, timeout=60
+ ) as response:
+ response.raise_for_status()
+ yield from response.iter_bytes(1024 * 1024)
+ return
+ path = Path(unquote(parsed.path)) if parsed.scheme == "file" else Path(source_url)
+ with path.open("rb") as stream:
+ yield from iter(lambda: stream.read(1024 * 1024), b"")
def _filename_from_url(source_url: str) -> str:
@@ -1492,6 +1522,8 @@ def _upload_r2_object(
wrangler_command: str,
) -> ArtifactCommandResult:
content_type, _ = mimetypes.guess_type(local_path.name)
+ if local_path.stat().st_size > WRANGLER_PUT_LIMIT_BYTES:
+ return _upload_r2_object_in_parts(location, local_path, content_type)
command = [
*shlex.split(wrangler_command),
"r2",
@@ -1507,6 +1539,61 @@ def _upload_r2_object(
return _run_command(command)
+def _upload_r2_object_in_parts(
+ location: ArtifactStorageLocation,
+ local_path: Path,
+ content_type: str | None,
+) -> ArtifactCommandResult:
+ """Upload a file too large for Wrangler through R2's S3 API (chronicle#409).
+
+ The key carries the file's sha256, so an object of the file's size under it
+ already holds these bytes. Such a file is not sent again, and a failed
+ upload, such as a completion whose response was lost, counts once the
+ object is there.
+ """
+ command = (
+ "r2-s3-multipart-put",
+ f"{location.bucket}/{location.key}",
+ str(local_path),
+ )
+ try:
+ size_bytes = local_path.stat().st_size
+ with _r2_s3_client() as client:
+ if client.object_size(location.bucket, location.key) == size_bytes:
+ return ArtifactCommandResult(
+ command=command, returncode=0, stdout="already in R2", stderr=""
+ )
+ try:
+ parts = client.put_file(
+ location.bucket, location.key, local_path, content_type=content_type
+ )
+ except r2_s3.R2S3Error as error:
+ try:
+ landed = client.object_size(location.bucket, location.key)
+ except r2_s3.R2S3Error:
+ landed = None
+ if landed != size_bytes:
+ raise error
+ return ArtifactCommandResult(
+ command=command,
+ returncode=0,
+ stdout=f"in R2 although the upload reported: {error}",
+ stderr="",
+ )
+ except (OSError, r2_s3.R2S3Error) as exc:
+ return ArtifactCommandResult(
+ command=command, returncode=1, stdout="", stderr=str(exc)
+ )
+ return ArtifactCommandResult(
+ command=command, returncode=0, stdout=f"uploaded in {parts} parts", stderr=""
+ )
+
+
+def _r2_s3_client() -> r2_s3.R2Client:
+ """Return the S3 API client for large uploads; tests substitute their own."""
+ return r2_s3.R2Client(r2_s3.credentials_from_environment())
+
+
def _r2_only_storage(spec: dict[str, Any]) -> tuple[bool, list[str]]:
"""Return whether a file entry keeps its bytes in R2 only, and any issues.
@@ -1578,9 +1665,7 @@ def _publish_raw_manifest_entry(
elif not artifact_path.exists():
errors.append("missing_file")
else:
- content = artifact_path.read_bytes()
- sha256_actual = hashlib.sha256(content).hexdigest()
- size_bytes = len(content)
+ sha256_actual, size_bytes = _hash_file(artifact_path)
if sha256_expected and sha256_actual != sha256_expected:
errors.append("checksum_mismatch")
@@ -1702,9 +1787,7 @@ def _inventory_entry(
if not r2_only:
errors.append("missing_file")
else:
- content = artifact_path.read_bytes()
- sha256_actual = hashlib.sha256(content).hexdigest()
- size_bytes = len(content)
+ sha256_actual, size_bytes = _hash_file(artifact_path)
if sha256_expected and sha256_actual != sha256_expected:
errors.append("checksum_mismatch")
storage = spec.get("storage") if isinstance(spec, dict) else None
diff --git a/chronicle/r2_s3.py b/chronicle/r2_s3.py
new file mode 100644
index 00000000..b1642610
--- /dev/null
+++ b/chronicle/r2_s3.py
@@ -0,0 +1,409 @@
+"""Upload large R2 objects through R2's S3-compatible API (chronicle#409).
+
+``wrangler r2 object put`` refuses objects above 300 MiB, so Chronicle sends
+larger files here. R2 accepts requests signed with AWS Signature Version 4 in
+region ``auto``. A file goes up as a multipart upload in fixed-size parts: each
+part is retried on its own, and the upload is aborted if a part still fails, so
+no partial object is left under the key.
+
+Credentials come from ``R2_ACCESS_KEY_ID`` and ``R2_SECRET_ACCESS_KEY`` when
+both are set, and otherwise from the ``CLOUDFLARE_API_TOKEN`` that Wrangler
+already uses: Cloudflare's R2 token documentation gives that token's S3 key ID
+as the token's ID and its secret as the SHA-256 of the token's value. The
+account is ``CLOUDFLARE_ACCOUNT_ID``, or else the ``account_id`` that
+``wrangler.toml`` pins.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+import gc
+import hashlib
+import hmac
+import os
+from pathlib import Path
+import time
+import tomllib
+from types import TracebackType
+from urllib.parse import quote
+from xml.etree import ElementTree
+from xml.sax.saxutils import escape
+
+import httpx
+
+
+#: R2 takes up to 10,000 parts per upload, so 100 MiB parts reach 976 GiB.
+DEFAULT_PART_SIZE_BYTES = 100 * 1024 * 1024
+CLOUDFLARE_API_URL = "https://api.cloudflare.com/client/v4"
+_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+
+
+class R2S3Error(RuntimeError):
+ """An R2 S3 API request or credential lookup failed."""
+
+
+@dataclass(frozen=True)
+class R2Credentials:
+ """An S3 API key pair for one Cloudflare account's R2 buckets."""
+
+ account_id: str
+ access_key_id: str
+ secret_access_key: str = field(repr=False)
+
+ @property
+ def host(self) -> str:
+ """Return the account's S3 API host."""
+ return f"{self.account_id}.r2.cloudflarestorage.com"
+
+
+def credentials_from_environment(
+ environ: Mapping[str, str] | None = None,
+ *,
+ wrangler_toml: str | Path = "wrangler.toml",
+ transport: httpx.BaseTransport | None = None,
+) -> R2Credentials:
+ """Return R2 S3 credentials from the environment (see the module docstring)."""
+ environ = os.environ if environ is None else environ
+ account_id = environ.get("CLOUDFLARE_ACCOUNT_ID") or _wrangler_account_id(
+ Path(wrangler_toml)
+ )
+ if not account_id:
+ raise R2S3Error(
+ "No Cloudflare account: set CLOUDFLARE_ACCOUNT_ID, or run from the "
+ "directory whose wrangler.toml pins account_id."
+ )
+ access_key_id = environ.get("R2_ACCESS_KEY_ID")
+ secret_access_key = environ.get("R2_SECRET_ACCESS_KEY")
+ if access_key_id and secret_access_key:
+ return R2Credentials(account_id, access_key_id, secret_access_key)
+ token = environ.get("CLOUDFLARE_API_TOKEN")
+ if not token:
+ raise R2S3Error(
+ "No R2 credentials: set R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY, "
+ "or CLOUDFLARE_API_TOKEN."
+ )
+ return R2Credentials(
+ account_id,
+ _api_token_id(token, account_id, transport=transport),
+ hashlib.sha256(token.encode("utf-8")).hexdigest(),
+ )
+
+
+def sign_v4(
+ method: str,
+ *,
+ host: str,
+ path: str,
+ query: Mapping[str, str],
+ headers: Mapping[str, str],
+ payload_sha256: str,
+ access_key_id: str,
+ secret_access_key: str,
+ now: datetime,
+ region: str = "auto",
+) -> dict[str, str]:
+ """Return a request's headers with its AWS Signature Version 4.
+
+ ``path`` is the request path already URI-encoded, as it is sent. Every
+ header returned is signed: ``host``, ``x-amz-content-sha256``,
+ ``x-amz-date`` and the ``headers`` given.
+ """
+ amz_date = now.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
+ signed = {
+ **{name.lower(): " ".join(value.split()) for name, value in headers.items()},
+ "host": host,
+ "x-amz-content-sha256": payload_sha256,
+ "x-amz-date": amz_date,
+ }
+ names = sorted(signed)
+ signed_names = ";".join(names)
+ canonical_request = "\n".join(
+ [
+ method,
+ path,
+ canonical_query(query),
+ "".join(f"{name}:{signed[name]}\n" for name in names),
+ signed_names,
+ payload_sha256,
+ ]
+ )
+ scope = f"{amz_date[:8]}/{region}/s3/aws4_request"
+ string_to_sign = "\n".join(
+ [
+ "AWS4-HMAC-SHA256",
+ amz_date,
+ scope,
+ hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
+ ]
+ )
+ key = f"AWS4{secret_access_key}".encode("utf-8")
+ for part in (amz_date[:8], region, "s3", "aws4_request"):
+ key = hmac.new(key, part.encode("utf-8"), hashlib.sha256).digest()
+ signature = hmac.new(
+ key, string_to_sign.encode("utf-8"), hashlib.sha256
+ ).hexdigest()
+ return {
+ **signed,
+ "authorization": (
+ f"AWS4-HMAC-SHA256 Credential={access_key_id}/{scope}, "
+ f"SignedHeaders={signed_names}, Signature={signature}"
+ ),
+ }
+
+
+def canonical_query(query: Mapping[str, str]) -> str:
+ """Return a query string in Signature Version 4's canonical form."""
+ return "&".join(
+ f"{_uri_encode(name)}={_uri_encode(value)}"
+ for name, value in sorted(query.items())
+ )
+
+
+class R2Client:
+ """Signed S3 API requests to one Cloudflare account's R2 buckets."""
+
+ def __init__(
+ self,
+ credentials: R2Credentials,
+ *,
+ transport: httpx.BaseTransport | None = None,
+ part_size: int = DEFAULT_PART_SIZE_BYTES,
+ attempts: int = 4,
+ clock: Callable[[], datetime] | None = None,
+ sleep: Callable[[float], None] = time.sleep,
+ ) -> None:
+ if part_size <= 0 or attempts <= 0:
+ raise ValueError("part_size and attempts must be positive")
+ self.credentials = credentials
+ self.part_size = part_size
+ self.attempts = attempts
+ self._clock = clock or (lambda: datetime.now(UTC))
+ self._sleep = sleep
+ self._http = httpx.Client(
+ transport=transport,
+ timeout=httpx.Timeout(60.0, read=600.0, write=600.0),
+ )
+
+ def __enter__(self) -> R2Client:
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
+ self.close()
+
+ def close(self) -> None:
+ """Close the HTTP connection pool."""
+ self._http.close()
+
+ def put_file(
+ self,
+ bucket: str,
+ key: str,
+ path: str | Path,
+ *,
+ content_type: str | None = None,
+ ) -> int:
+ """Upload a file as a multipart upload and return its part count.
+
+ Parts are read one at a time, so memory holds one part, not the file.
+ """
+ path = Path(path)
+ if path.stat().st_size == 0:
+ raise R2S3Error(f"A multipart upload needs a non-empty file: {path}")
+ upload_id = self._create_upload(bucket, key, content_type)
+ parts: list[tuple[int, str]] = []
+ try:
+ with path.open("rb") as stream:
+ while chunk := stream.read(self.part_size):
+ number = len(parts) + 1
+ parts.append(
+ (
+ number,
+ self._upload_part(bucket, key, upload_id, number, chunk),
+ )
+ )
+ # httpx keeps each response in a reference cycle with its
+ # request, which holds the part it sent. The cycle
+ # collector can leave those for many parts, and so most of
+ # a file in memory, so free the part before the next read.
+ del chunk
+ gc.collect()
+ self._complete_upload(bucket, key, upload_id, parts)
+ except BaseException:
+ self._abort_upload(bucket, key, upload_id)
+ raise
+ return len(parts)
+
+ def object_size(self, bucket: str, key: str) -> int | None:
+ """Return the size in bytes of the object under ``key``, or None if absent."""
+ response = self._request("HEAD", bucket, key, {}, missing_ok=True)
+ if response.status_code == 404:
+ return None
+ size = response.headers.get("content-length")
+ if size is None:
+ raise R2S3Error(f"HEAD {key} returned no Content-Length")
+ return int(size)
+
+ def _create_upload(self, bucket: str, key: str, content_type: str | None) -> str:
+ response = self._request(
+ "POST",
+ bucket,
+ key,
+ {"uploads": ""},
+ headers={"content-type": content_type} if content_type else {},
+ )
+ upload_id = _xml_text(response, "UploadId")
+ if not upload_id:
+ raise R2S3Error(f"CreateMultipartUpload for {key} returned no UploadId")
+ return upload_id
+
+ def _upload_part(
+ self, bucket: str, key: str, upload_id: str, number: int, chunk: bytes
+ ) -> str:
+ response = self._request(
+ "PUT",
+ bucket,
+ key,
+ {"partNumber": str(number), "uploadId": upload_id},
+ body=chunk,
+ )
+ etag = response.headers.get("etag")
+ if not etag:
+ raise R2S3Error(f"UploadPart {number} of {key} returned no ETag")
+ return etag
+
+ def _complete_upload(
+ self, bucket: str, key: str, upload_id: str, parts: list[tuple[int, str]]
+ ) -> None:
+ body = "".join(
+ f"{number}{escape(etag)}"
+ for number, etag in parts
+ )
+ response = self._request(
+ "POST",
+ bucket,
+ key,
+ {"uploadId": upload_id},
+ body=f"{body}".encode(
+ "utf-8"
+ ),
+ headers={"content-type": "application/xml"},
+ )
+ # S3 can report a failed completion inside a 200 response.
+ if _xml_root_tag(response) == "Error":
+ raise R2S3Error(
+ f"CompleteMultipartUpload for {key} failed: {response.text[:300]}"
+ )
+
+ def _abort_upload(self, bucket: str, key: str, upload_id: str) -> None:
+ try:
+ self._request("DELETE", bucket, key, {"uploadId": upload_id})
+ except R2S3Error:
+ # The caller's error matters more; R2 expires abandoned uploads.
+ pass
+
+ def _request(
+ self,
+ method: str,
+ bucket: str,
+ key: str,
+ query: Mapping[str, str],
+ *,
+ body: bytes = b"",
+ headers: Mapping[str, str] | None = None,
+ missing_ok: bool = False,
+ ) -> httpx.Response:
+ path = f"/{_uri_encode(bucket)}/{_uri_encode(key, keep_slash=True)}"
+ url = f"https://{self.credentials.host}{path}"
+ if query:
+ url = f"{url}?{canonical_query(query)}"
+ payload_sha256 = hashlib.sha256(body).hexdigest()
+ for attempt in range(1, self.attempts + 1):
+ signed = sign_v4(
+ method,
+ host=self.credentials.host,
+ path=path,
+ query=query,
+ headers=headers or {},
+ payload_sha256=payload_sha256,
+ access_key_id=self.credentials.access_key_id,
+ secret_access_key=self.credentials.secret_access_key,
+ now=self._clock(),
+ )
+ try:
+ response = self._http.request(method, url, content=body, headers=signed)
+ except httpx.TransportError as exc:
+ if attempt == self.attempts:
+ raise R2S3Error(f"{method} {key} failed: {exc}") from exc
+ else:
+ if response.is_success or (missing_ok and response.status_code == 404):
+ return response
+ if (
+ response.status_code not in _RETRY_STATUSES
+ or attempt == self.attempts
+ ):
+ raise R2S3Error(
+ f"{method} {key} returned HTTP {response.status_code}: "
+ f"{response.text[:300]}"
+ )
+ self._sleep(2.0**attempt)
+ raise AssertionError("unreachable")
+
+
+def _api_token_id(
+ token: str, account_id: str, *, transport: httpx.BaseTransport | None
+) -> str:
+ """Return an API token's ID: user tokens and account tokens verify apart."""
+ try:
+ with httpx.Client(transport=transport, timeout=30.0) as client:
+ for url in (
+ f"{CLOUDFLARE_API_URL}/user/tokens/verify",
+ f"{CLOUDFLARE_API_URL}/accounts/{account_id}/tokens/verify",
+ ):
+ response = client.get(url, headers={"Authorization": f"Bearer {token}"})
+ if response.status_code == 200:
+ token_id = (response.json().get("result") or {}).get("id")
+ if token_id:
+ return str(token_id)
+ except (httpx.HTTPError, ValueError) as exc:
+ raise R2S3Error(f"Could not verify CLOUDFLARE_API_TOKEN: {exc}") from exc
+ raise R2S3Error("CLOUDFLARE_API_TOKEN did not verify, so its R2 key ID is unknown.")
+
+
+def _wrangler_account_id(path: Path) -> str | None:
+ try:
+ payload = tomllib.loads(path.read_text(encoding="utf-8"))
+ except (OSError, tomllib.TOMLDecodeError):
+ return None
+ account_id = payload.get("account_id")
+ return account_id if isinstance(account_id, str) and account_id else None
+
+
+def _uri_encode(value: str, *, keep_slash: bool = False) -> str:
+ return quote(value, safe="-_.~/" if keep_slash else "-_.~")
+
+
+def _xml_text(response: httpx.Response, tag: str) -> str | None:
+ try:
+ root = ElementTree.fromstring(response.content)
+ except ElementTree.ParseError:
+ return None
+ # ``{*}`` matches the tag in S3's namespace or in none.
+ element = root.find(f".//{{*}}{tag}")
+ return element.text if element is not None else None
+
+
+def _xml_root_tag(response: httpx.Response) -> str | None:
+ if not response.content.strip():
+ return None
+ try:
+ tag = ElementTree.fromstring(response.content).tag
+ except ElementTree.ParseError:
+ return None
+ return tag.rsplit("}", 1)[-1]
diff --git a/db/data/ucl/house_price_per_square_metre_856542/manifest.yaml b/db/data/ucl/house_price_per_square_metre_856542/manifest.yaml
new file mode 100644
index 00000000..e4392f25
--- /dev/null
+++ b/db/data/ucl/house_price_per_square_metre_856542/manifest.yaml
@@ -0,0 +1,65 @@
+source_id: ucl
+package_id: ucl-house-price-per-square-metre-856542
+dataset: ucl_ucl-house-price-per-square-metre-856542
+source_name: House Price Per Square Metre in England and Wales, 1995-2023
+publisher: University College London, deposited in UK Data Service ReShare
+source_page: https://reshare.ukdataservice.ac.uk/856542/
+table: UK Data Service ReShare 856542, bundles 1 and 2
+license: Creative Commons Attribution 4.0 International (CC BY 4.0)
+license_url: https://creativecommons.org/licenses/by/4.0/
+attribution: Chi, B., Dennett, A., Oléron-Evans, T. and Morphet, R. (2025). House Price Per Square
+ Metre in England and Wales, 1995-2023. UK Data Service ReShare. DOI 10.5255/UKDA-SN-856542
+license_evidence: 'ReShare''s record for 856542 states Creative Commons Attribution 4.0 International
+ (CC BY 4.0). UCL''s Licence_Information_for_data_sources.docx in bundle 1 adds that tranall_link_02122023.zip,
+ LRPPD_02122023.zip and NSPL_NOV_2023_UK.zip contain Royal Mail copyright postcode and address data,
+ limited to non-commercial use, with all non-address data under the Open Government Licence v3.0.
+ It quotes the EPC terms, which say the data contains personal data and that Royal Mail''s copyright
+ notice must go with any address data that is shared, and the Price Paid terms. So the bytes stay
+ out of this public repository: the zips are git-ignored and the private ledger-raw bucket holds
+ them, bundle 1 with that notice.'
+license_verified_at: '2026-11-06'
+files:
+ bundle_1:
+ filename: 856542_bundle_1.zip
+ source_url: https://reshare.ukdataservice.ac.uk/856542/1/856542_bundle_1.zip
+ source_table: 'ReShare 856542 bundle 1: the linked file tranall_link_02122023.zip, with UCL''s
+ readme and licence note'
+ sha256: e397cdac214b918a39f3ccf5266adaf9e4c3d8fb4f71e0bdc1b9c9474734b296
+ size_bytes: 3214161126
+ fetched_at: '2026-11-06T11:17:39+00:00'
+ storage:
+ local: false
+ r2:
+ provider: r2
+ bucket: ledger-raw
+ key: raw/uk/ucl/ucl-house-price-per-square-metre-856542/bundle_1/e397cdac214b918a39f3ccf5266adaf9e4c3d8fb4f71e0bdc1b9c9474734b296/856542_bundle_1.zip
+ uri: r2://ledger-raw/raw/uk/ucl/ucl-house-price-per-square-metre-856542/bundle_1/e397cdac214b918a39f3ccf5266adaf9e4c3d8fb4f71e0bdc1b9c9474734b296/856542_bundle_1.zip
+ bundle_2:
+ filename: 856542_bundle_2.zip
+ source_url: https://reshare.ukdataservice.ac.uk/856542/2/856542_bundle_2.zip
+ source_table: 'ReShare 856542 bundle 2: the files UCL linked, epc20_id.zip (domestic EPCs with
+ addresses removed), LRPPD_02122023.zip (Price Paid as at 2 December 2023) and NSPL_NOV_2023_UK.zip'
+ sha256: 61cbf6a72e35f7569d9121d80eaf89e757fec3bd4c21f1175e37fff44e02da97
+ size_bytes: 4426805029
+ fetched_at: '2026-11-06T11:14:45+00:00'
+ storage:
+ local: false
+ r2:
+ provider: r2
+ bucket: ledger-raw
+ key: raw/uk/ucl/ucl-house-price-per-square-metre-856542/bundle_2/61cbf6a72e35f7569d9121d80eaf89e757fec3bd4c21f1175e37fff44e02da97/856542_bundle_2.zip
+ uri: r2://ledger-raw/raw/uk/ucl/ucl-house-price-per-square-metre-856542/bundle_2/61cbf6a72e35f7569d9121d80eaf89e757fec3bd4c21f1175e37fff44e02da97/856542_bundle_2.zip
+registration_kind: raw_only
+provenance_notes: 'UK Data Service ReShare 856542 (DOI 10.5255/UKDA-SN-856542), published on 22 August
+ 2025 by Bin Chi, Adam Dennett, Thomas Oléron-Evans and Robin Morphet of UCL, the latest deposit
+ of the series 854240, 855033, 856204 and 856542. Bundle 1 holds the linked file tranall_link_02122023.zip
+ (3,214,108,822 bytes, stored), which UCL''s readme describes as 21,438,854 transactions in England
+ and Wales from 1 January 1995 to 31 October 2023 with 107 fields, plus the readme and the licence
+ note. ReShare''s record gives the temporal coverage as 1 January 1995 to 23 October 2023, while
+ its abstract, like the readme, says 31 October 2023. Bundle 2 holds the files UCL linked: the domestic
+ EPCs downloaded on 2 December 2023 with their address fields removed (epc20_id.zip), Price Paid
+ with field names as at 2 December 2023 (LRPPD_02122023.zip) and the November 2023 NSPL (NSPL_NOV_2023_UK.zip).
+ ReShare''s ETags equal the MD5s of the bytes fetched on 6 November 2026 (239aebb08e10939b7cba4c5b2d5f89c9
+ and 5fa5f769f3061aef86c455a4ed23c97e), and both bundles were last modified on 22 August 2025. Raw
+ only: no build parses either bundle (chronicle#408, item 6). Both are above the 300 MiB that wrangler
+ r2 object put accepts, so publish-raw uploads them through R2''s S3 API in 100 MiB parts (chronicle#409).'
diff --git a/docs/agent-source-package-harness.md b/docs/agent-source-package-harness.md
index 261c1dd5..de1e56c3 100644
--- a/docs/agent-source-package-harness.md
+++ b/docs/agent-source-package-harness.md
@@ -19,7 +19,13 @@ metadata in `manifest.yaml`, and can upload the exact bytes to the private
artifact registry with `uv run chronicle inventory-artifacts --root db/data`.
For already-downloaded manifest artifacts, agents should run
`uv run chronicle publish-raw --root db/data` to upload checksum-verified bytes to
-R2 and write `storage.r2` metadata back into each manifest entry. A manifest
+R2 and write `storage.r2` metadata back into each manifest entry. A file above the
+300 MiB that `wrangler r2 object put` accepts goes through R2's S3-compatible API
+instead, as a multipart upload in 100 MiB parts
+([chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409)). That path
+signs with `R2_ACCESS_KEY_ID` and `R2_SECRET_ACCESS_KEY` when both are set, and
+otherwise with the S3 key pair Cloudflare derives from `CLOUDFLARE_API_TOKEN`. It
+takes the account from `CLOUDFLARE_ACCOUNT_ID` or `wrangler.toml`. A manifest
entry marked `storage.local: false` keeps its bytes in R2 only and never in Git:
both commands accept it without a local copy, so its absence is not an error to
fix by downloading, and `uv run chronicle verify-raw` checks the R2 object.
diff --git a/docs/data-sources/uk-high-value-housing.md b/docs/data-sources/uk-high-value-housing.md
index bc7af85c..ade857d1 100644
--- a/docs/data-sources/uk-high-value-housing.md
+++ b/docs/data-sources/uk-high-value-housing.md
@@ -72,9 +72,11 @@ Each of their file entries says so with `storage.local: false`:
`.gitignore` keeps the local copies out of commits.
Price Paid is registered as HM Land Registry's yearly files, not its single
-`pp-complete.csv` (5,525,539,417 bytes). `publish-raw` uploads through
+`pp-complete.csv` (5,525,539,417 bytes). `publish-raw` then uploaded only through
`wrangler r2 object put`, which refuses files above 300 MiB; the largest yearly
-file (`pp-2002.csv`) is 239,344,429 bytes. The 32 yearly files of a release hold
+file (`pp-2002.csv`) is 239,344,429 bytes. Larger files now go through R2's S3
+API ([chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409)), but
+the yearly files stay: they are the release's own split. The 32 yearly files of a release hold
exactly the complete file's rows, split by year of transfer. For this release,
order-independent row digests by year agree between the two (31,610,742 rows).
diff --git a/docs/data-sources/uk-land-value-inputs-408.md b/docs/data-sources/uk-land-value-inputs-408.md
index 29a6a5e6..1846232c 100644
--- a/docs/data-sources/uk-land-value-inputs-408.md
+++ b/docs/data-sources/uk-land-value-inputs-408.md
@@ -396,13 +396,46 @@ These rows give Microcosm council tax by authority for all three nations, which
- **Table 9a against Tables 9d and 2e.** 9a's 155,400 claims equal 9d's total. 2e's quarters
from 2024 Q2 to 2025 Q1 sum to 155,368.45.
-## 6. UCL's house price per square metre: not in this PR
-
-ReShare 856542's two bundles are 3,214,161,126 and 4,426,805,029 bytes. Each holds stored
-nested zips, the linked file `tranall_link_02122023.zip` alone being 3.2 GB. `publish-raw`
-uploads through `wrangler r2 object put`, which refuses objects over 300 MiB. Item 6 therefore
-waits for a separate change to how Chronicle stores large raw files, which
-[chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409) asks for a ruling on.
+## 6. UCL's house price per square metre: both bundles, raw-only
+
+**`ucl-house-price-per-square-metre-856542`** registers
+[UK Data Service ReShare 856542](https://reshare.ukdataservice.ac.uk/856542/), *House Price
+Per Square Metre in England and Wales, 1995-2023*. It is by Bin Chi, Adam Dennett, Thomas
+Oléron-Evans and Robin Morphet (UCL), published on 22 August 2025 under CC BY 4.0. No build
+parses it. It came after the rest of this issue, once Chronicle could store files above
+300 MiB ([chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409)).
+
+- **Bundle 1**, `856542_bundle_1.zip`: 3,214,161,126 bytes, sha256 `e397cdac…34b296`.
+ - It holds the linked file `tranall_link_02122023.zip` (3,214,108,822 bytes, stored), with
+ UCL's readme and licence note.
+ - The readme describes the linked file as 21,438,854 transactions in England and Wales from
+ 1 January 1995 to 31 October 2023, with 107 fields.
+ - ReShare's record gives the temporal coverage as 1 January 1995 to 23 October 2023, while
+ its abstract, like the readme, says 31 October 2023.
+- **Bundle 2**, `856542_bundle_2.zip`: 4,426,805,029 bytes, sha256 `61cbf6a7…02da97`. It
+ holds the files UCL linked:
+ - the domestic EPCs downloaded on 2 December 2023, with their address fields removed
+ (`epc20_id.zip`);
+ - Price Paid with field names as at 2 December 2023 (`LRPPD_02122023.zip`);
+ - the November 2023 NSPL (`NSPL_NOV_2023_UK.zip`).
+- ReShare's ETags equal the MD5s of the bytes fetched on 6 November 2026, and both bundles
+ were last modified on 22 August 2025.
+
+**Licence.** ReShare states CC BY 4.0. UCL's licence note adds that the linked file, Price
+Paid and the NSPL carry Royal Mail address data, limited to non-commercial use, with
+everything else under the Open Government Licence v3.0. It also quotes the EPC terms: the data
+contains personal data, and Royal Mail's notice must go with any address data that is shared.
+As with Price Paid, the zips are git-ignored and the private `ledger-raw` bucket holds them.
+
+**Storage.** Both bundles are above the 300 MiB that `wrangler r2 object put` accepts, so
+`publish-raw` sent them through R2's S3 API, in 31 and 43 parts of 100 MiB. R2's ETags for
+them, `"7eb31fba…44b1df-31"` and `"d04abcef…798c87-43"`, are the MD5 of the parts' MD5s,
+which can be recomputed from the pinned bytes without downloading the objects. `verify-raw`
+reads them back in full through `wrangler r2 object get --pipe`, which streams.
+
+Microcosm needs bundle 1's linked file for each sale's floor area; bundle 2 keeps what UCL
+linked from. The London Datastore's 2024 edition is not registered, because its record states
+no licence.
## 7. Floor area and dwelling density by authority
diff --git a/docs/pe-uk-source-checklist.md b/docs/pe-uk-source-checklist.md
index e403b860..4b7666d1 100644
--- a/docs/pe-uk-source-checklist.md
+++ b/docs/pe-uk-source-checklist.md
@@ -457,7 +457,9 @@ facts; field contracts and caveats are in
Published inputs for Microcosm's land value work
([PolicyEngine/uk-land-value-tax](https://github.com/PolicyEngine/uk-land-value-tax)). Eight of
-the issue's nine items are here: about 326,500 new facts, plus three raw-only registrations.
+the issue's nine items came first: about 326,500 new facts, plus three raw-only registrations.
+The ninth, UCL's house price per square metre, followed once `publish-raw` could upload files
+above 300 MiB ([chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409)).
Field contracts and caveats are in
[`docs/data-sources/uk-land-value-inputs-408.md`](data-sources/uk-land-value-inputs-408.md).
@@ -484,8 +486,8 @@ Field contracts and caveats are in
`hmrc-stamp-tax-statistics-2024-25` (1,135 facts).
- **Tax Policy Associates' model.** Three files pinned raw at one commit, with no facts
(`tpa-lvt-model-2026`).
-- **Not in this PR.** UCL's house price per square metre file (item 6) is above the 300 MiB that
- `publish-raw` can upload ([chronicle#409](https://github.com/PolicyEngine/chronicle/issues/409)).
+- **UCL's house price per square metre (item 6).** UK Data Service ReShare 856542's two
+ bundles, 3.2 and 4.4 GB, pinned raw with no facts (`ucl-house-price-per-square-metre-856542`).
## Consumer contracts
diff --git a/tests/test_chronicle_artifacts.py b/tests/test_chronicle_artifacts.py
index ff407ec5..733aad6d 100644
--- a/tests/test_chronicle_artifacts.py
+++ b/tests/test_chronicle_artifacts.py
@@ -13,6 +13,9 @@
from chronicle.cli import main as cli_main
from chronicle.artifacts import (
R2_COUNTRY_PUBLISHERS,
+ WRANGLER_PUT_LIMIT_BYTES,
+ ArtifactStorageLocation,
+ _upload_r2_object,
build_artifact_key,
build_artifact_rows,
build_derived_r2_key,
@@ -28,6 +31,7 @@
from chronicle.bundle import NZ_BUNDLE_SOURCE_PREFIXES, NZ_BUNDLE_SOURCES
from chronicle.epoch import Epoch
from chronicle.harness import main as harness_main
+from chronicle.r2_s3 import R2S3Error
from chronicle.source_package import load_source_package
@@ -677,6 +681,213 @@ def test_publish_raw_refuses_r2_only_bytes_without_an_r2_location(tmp_path):
assert not log.exists()
+def _fetch_release(tmp_path: Path, content: bytes) -> Path:
+ source = tmp_path / "release.csv"
+ source.write_bytes(content)
+ output_dir = tmp_path / "data" / "hmlr" / "release"
+ fetch_source_artifact(
+ str(source),
+ source_id="hmlr",
+ package_id="hmlr-release",
+ year=2026,
+ output_dir=output_dir,
+ )
+ return output_dir
+
+
+def test_fetch_publish_and_inventory_hash_files_without_reading_them_whole(
+ tmp_path, monkeypatch
+):
+ # Raw files can be several GB (chronicle#409), so none is read whole.
+ def refuse_whole_read(path):
+ raise AssertionError(f"read whole: {path}")
+
+ monkeypatch.setattr(Path, "read_bytes", refuse_whole_read)
+ wrangler, _ = _logging_wrangler(tmp_path)
+ output_dir = _fetch_release(tmp_path, b"raw release bytes")
+
+ published = publish_source_artifacts(output_dir, wrangler_command=str(wrangler))
+ inventory = inventory_source_artifacts(output_dir)
+
+ assert published.valid and inventory.valid
+ (entry,) = published.entries
+ assert entry.sha256 == hashlib.sha256(b"raw release bytes").hexdigest()
+ assert entry.size_bytes == len(b"raw release bytes")
+ assert inventory.entries[0].sha256_actual == entry.sha256
+
+
+class _RecordingS3Client:
+ uploads: list[tuple[str, str, bytes, str | None]] = []
+ error: Exception | None = None
+ # What HEAD finds under the key before an upload, and after one that failed.
+ existing_size: int | Exception | None = None
+ size_after_error: int | Exception | None = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc_info):
+ return None
+
+ def object_size(self, bucket, key):
+ if isinstance(self.existing_size, Exception):
+ raise self.existing_size
+ return self.existing_size
+
+ def put_file(self, bucket, key, path, *, content_type=None):
+ if self.error is not None:
+ type(self).existing_size = self.size_after_error
+ raise self.error
+ self.uploads.append((bucket, key, Path(path).read_bytes(), content_type))
+ return 1
+
+
+def test_files_above_the_wrangler_limit_go_through_the_s3_api(tmp_path, monkeypatch):
+ wrangler, log = _logging_wrangler(tmp_path)
+ output_dir = _fetch_release(tmp_path, b"above the limit")
+ monkeypatch.setattr("chronicle.artifacts.WRANGLER_PUT_LIMIT_BYTES", 4)
+ monkeypatch.setattr(_RecordingS3Client, "uploads", [])
+ monkeypatch.setattr("chronicle.artifacts._r2_s3_client", _RecordingS3Client)
+
+ report = publish_source_artifacts(output_dir, wrangler_command=str(wrangler))
+ manifest = yaml.safe_load((output_dir / "manifest.yaml").read_text())
+ r2 = manifest["files"][2026]["storage"]["r2"]
+
+ assert report.valid
+ assert report.counts["uploaded_count"] == 1
+ assert report.entries[0].upload.command[0] == "r2-s3-multipart-put"
+ assert _RecordingS3Client.uploads == [
+ ("ledger-raw", r2["key"], b"above the limit", "text/csv")
+ ]
+ assert r2["key"].startswith("raw/uk/hmlr/hmlr-release/2026/")
+ assert not log.exists()
+
+
+@pytest.mark.parametrize(
+ "size_after_error",
+ [None, R2S3Error("HEAD failed")],
+ ids=["nothing-under-the-key", "head-fails-too"],
+)
+def test_a_failed_s3_upload_records_no_r2_location(
+ tmp_path, monkeypatch, size_after_error
+):
+ wrangler, log = _logging_wrangler(tmp_path)
+ output_dir = _fetch_release(tmp_path, b"above the limit")
+ monkeypatch.setattr("chronicle.artifacts.WRANGLER_PUT_LIMIT_BYTES", 4)
+ monkeypatch.setattr(_RecordingS3Client, "error", R2S3Error("HTTP 403: denied"))
+ monkeypatch.setattr(_RecordingS3Client, "existing_size", None)
+ monkeypatch.setattr(_RecordingS3Client, "size_after_error", size_after_error)
+ monkeypatch.setattr("chronicle.artifacts._r2_s3_client", _RecordingS3Client)
+
+ report = publish_source_artifacts(output_dir, wrangler_command=str(wrangler))
+ manifest = yaml.safe_load((output_dir / "manifest.yaml").read_text())
+
+ assert not report.valid
+ assert report.entries[0].errors == ("r2_upload_failed",)
+ # The upload's own error is reported, not the check that followed it.
+ assert report.entries[0].upload.stderr == "HTTP 403: denied"
+ assert "storage" not in manifest["files"][2026]
+ assert not log.exists()
+
+
+@pytest.mark.parametrize(
+ ("existing_size", "sent"),
+ [(15, False), (14, True)],
+ ids=["same-size", "other-size"],
+)
+def test_an_object_of_the_files_size_under_its_key_is_not_sent_again(
+ tmp_path, monkeypatch, existing_size, sent
+):
+ # The key carries the file's sha256, so an object of its size is the file.
+ wrangler, log = _logging_wrangler(tmp_path)
+ output_dir = _fetch_release(tmp_path, b"above the limit")
+ monkeypatch.setattr("chronicle.artifacts.WRANGLER_PUT_LIMIT_BYTES", 4)
+ monkeypatch.setattr(_RecordingS3Client, "uploads", [])
+ monkeypatch.setattr(_RecordingS3Client, "existing_size", existing_size)
+ monkeypatch.setattr("chronicle.artifacts._r2_s3_client", _RecordingS3Client)
+
+ report = publish_source_artifacts(output_dir, wrangler_command=str(wrangler))
+ manifest = yaml.safe_load((output_dir / "manifest.yaml").read_text())
+ entry = report.entries[0]
+
+ assert report.valid
+ assert manifest["files"][2026]["storage"]["r2"]["key"] == entry.r2_location.key
+ assert len(_RecordingS3Client.uploads) == int(sent)
+ assert (entry.upload.stdout == "already in R2") is not sent
+ assert not log.exists()
+
+
+def test_an_upload_whose_completion_landed_counts_as_uploaded(tmp_path, monkeypatch):
+ # A completion can commit and lose its response: the retry then finds no
+ # upload and put_file raises, though the object is in R2.
+ wrangler, log = _logging_wrangler(tmp_path)
+ output_dir = _fetch_release(tmp_path, b"above the limit")
+ monkeypatch.setattr("chronicle.artifacts.WRANGLER_PUT_LIMIT_BYTES", 4)
+ monkeypatch.setattr(
+ _RecordingS3Client,
+ "error",
+ R2S3Error("POST key returned HTTP 404: NoSuchUpload"),
+ )
+ monkeypatch.setattr(_RecordingS3Client, "existing_size", None)
+ monkeypatch.setattr(_RecordingS3Client, "size_after_error", 15)
+ monkeypatch.setattr("chronicle.artifacts._r2_s3_client", _RecordingS3Client)
+
+ report = publish_source_artifacts(output_dir, wrangler_command=str(wrangler))
+ manifest = yaml.safe_load((output_dir / "manifest.yaml").read_text())
+ entry = report.entries[0]
+
+ assert report.valid
+ assert entry.errors == ()
+ assert "NoSuchUpload" in entry.upload.stdout
+ assert manifest["files"][2026]["storage"]["r2"]["key"] == entry.r2_location.key
+ assert not log.exists()
+
+
+def test_the_s3_path_starts_one_byte_above_the_wrangler_limit(tmp_path, monkeypatch):
+ # Wrangler refuses only files above its limit (sizeBytes > 300 MiB).
+ wrangler, log = _logging_wrangler(tmp_path)
+ sent: list[int] = []
+
+ class SizeRecordingClient(_RecordingS3Client):
+ def put_file(self, bucket, key, path, *, content_type=None):
+ sent.append(Path(path).stat().st_size)
+ return 4
+
+ monkeypatch.setattr(_RecordingS3Client, "existing_size", None)
+ monkeypatch.setattr("chronicle.artifacts._r2_s3_client", SizeRecordingClient)
+ location = ArtifactStorageLocation(provider="r2", bucket="ledger-raw", key="raw/k")
+ results = {}
+ for size in (WRANGLER_PUT_LIMIT_BYTES, WRANGLER_PUT_LIMIT_BYTES + 1):
+ path = tmp_path / f"{size}.zip"
+ with path.open("wb") as stream:
+ stream.truncate(size) # sparse, so nothing is written
+ results[size] = _upload_r2_object(
+ location, path, wrangler_command=str(wrangler)
+ )
+
+ assert WRANGLER_PUT_LIMIT_BYTES == 300 * 1024 * 1024
+ assert results[WRANGLER_PUT_LIMIT_BYTES].command[0] == str(wrangler)
+ assert results[WRANGLER_PUT_LIMIT_BYTES + 1].command[0] == "r2-s3-multipart-put"
+ assert sent == [WRANGLER_PUT_LIMIT_BYTES + 1]
+ assert log.read_text().count("r2 object put") == 1
+
+
+def test_fetch_keeps_the_earlier_copy_when_a_download_fails(tmp_path):
+ output_dir = _fetch_release(tmp_path, b"first copy")
+
+ with pytest.raises(FileNotFoundError):
+ fetch_source_artifact(
+ str(tmp_path / "gone" / "release.csv"),
+ source_id="hmlr",
+ package_id="hmlr-release",
+ year=2026,
+ output_dir=output_dir,
+ )
+
+ assert (output_dir / "release.csv").read_bytes() == b"first copy"
+ assert not (output_dir / "release.csv.partial").exists()
+
+
def test_artifact_cli_commands_emit_json(tmp_path, capsys):
source = tmp_path / "source.xls"
source.write_bytes(b"cli artifact")
diff --git a/tests/test_chronicle_r2_s3.py b/tests/test_chronicle_r2_s3.py
new file mode 100644
index 00000000..6f9acce9
--- /dev/null
+++ b/tests/test_chronicle_r2_s3.py
@@ -0,0 +1,374 @@
+"""Tests for Chronicle's large-file path to R2 through its S3 API (chronicle#409)."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+import gc
+import hashlib
+from pathlib import Path
+import tracemalloc
+
+import httpx
+import pytest
+
+from chronicle.r2_s3 import (
+ R2Client,
+ R2Credentials,
+ R2S3Error,
+ canonical_query,
+ credentials_from_environment,
+ sign_v4,
+)
+
+
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+CREDENTIALS = R2Credentials("account", "key-id", "secret")
+
+
+@pytest.mark.parametrize(
+ ("method", "path", "query", "headers", "payload", "signature"),
+ [
+ # The three Signature Version 4 examples in AWS's S3 API reference
+ # ("Authenticating Requests: Using the Authorization Header").
+ (
+ "GET",
+ "/test.txt",
+ {},
+ {"Range": "bytes=0-9"},
+ b"",
+ "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41",
+ ),
+ (
+ "GET",
+ "/",
+ {"max-keys": "2", "prefix": "J"},
+ {},
+ b"",
+ "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7",
+ ),
+ (
+ "PUT",
+ "/test%24file.text",
+ {},
+ {
+ "Date": "Fri, 24 May 2013 00:00:00 GMT",
+ "x-amz-storage-class": "REDUCED_REDUNDANCY",
+ },
+ b"Welcome to Amazon S3.",
+ "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd",
+ ),
+ ],
+)
+def test_sign_v4_reproduces_the_aws_examples(
+ method, path, query, headers, payload, signature
+):
+ signed = sign_v4(
+ method,
+ host="examplebucket.s3.amazonaws.com",
+ path=path,
+ query=query,
+ headers=headers,
+ payload_sha256=hashlib.sha256(payload).hexdigest(),
+ access_key_id="AKIAIOSFODNN7EXAMPLE",
+ secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ now=datetime(2013, 5, 24, tzinfo=UTC),
+ region="us-east-1",
+ )
+
+ assert signed["authorization"].endswith(f"Signature={signature}")
+ assert signed["authorization"].startswith(
+ "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/"
+ "aws4_request, SignedHeaders="
+ )
+
+
+def test_canonical_query_sorts_names_and_encodes_values():
+ assert canonical_query({"uploadId": "a+b/c=", "partNumber": "2"}) == (
+ "partNumber=2&uploadId=a%2Bb%2Fc%3D"
+ )
+ assert canonical_query({"uploads": ""}) == "uploads="
+
+
+class FakeR2:
+ """An in-memory S3 multipart endpoint that records every request."""
+
+ def __init__(self, fail_part: dict[int, list[int]] | None = None):
+ self.requests: list[httpx.Request] = []
+ self.parts: dict[int, bytes] = {}
+ self.fail_part = fail_part or {}
+ self.complete_status = 200
+ self.complete_body = b""
+ self.objects: dict[str, int] = {}
+ self.head_status: int | None = None
+
+ def handler(self, request: httpx.Request) -> httpx.Response:
+ self.requests.append(request)
+ query = dict(request.url.params)
+ assert (
+ request.headers["x-amz-content-sha256"]
+ == hashlib.sha256(request.content).hexdigest()
+ )
+ assert request.headers["authorization"].startswith(
+ "AWS4-HMAC-SHA256 Credential=key-id/"
+ )
+ if request.method == "HEAD":
+ if self.head_status is not None:
+ return httpx.Response(self.head_status)
+ key = request.url.path.split("/", 2)[2]
+ if key not in self.objects:
+ return httpx.Response(404)
+ return httpx.Response(
+ 200, headers={"Content-Length": str(self.objects[key])}
+ )
+ if request.method == "POST" and "uploads" in query:
+ return httpx.Response(
+ 200,
+ content=(
+ b'up+1/='
+ b""
+ ),
+ )
+ if request.method == "PUT":
+ number = int(query["partNumber"])
+ statuses = self.fail_part.get(number, [])
+ if statuses:
+ return httpx.Response(statuses.pop(0))
+ self.parts[number] = request.content
+ return httpx.Response(200, headers={"ETag": f'"etag-{number}"'})
+ if request.method == "POST":
+ return httpx.Response(self.complete_status, content=self.complete_body)
+ if request.method == "DELETE":
+ return httpx.Response(204)
+ raise AssertionError(f"unexpected request {request.method} {request.url}")
+
+ def client(self, **kwargs) -> R2Client:
+ return R2Client(
+ CREDENTIALS,
+ transport=httpx.MockTransport(self.handler),
+ part_size=5,
+ sleep=lambda seconds: None,
+ **kwargs,
+ )
+
+
+def _file(tmp_path: Path, content: bytes) -> Path:
+ path = tmp_path / "bundle.zip"
+ path.write_bytes(content)
+ return path
+
+
+def test_put_file_uploads_fixed_size_parts_and_completes_them_in_order(tmp_path):
+ fake = FakeR2()
+
+ with fake.client() as client:
+ parts = client.put_file(
+ "ledger-raw",
+ "raw/uk/ucl/pkg/bundle_1/digest/bundle.zip",
+ _file(tmp_path, b"abcdefghijkl"),
+ content_type="application/zip",
+ )
+
+ assert parts == 3
+ assert fake.parts == {1: b"abcde", 2: b"fghij", 3: b"kl"}
+ create, *uploads, complete = fake.requests
+ assert create.url.path == "/ledger-raw/raw/uk/ucl/pkg/bundle_1/digest/bundle.zip"
+ assert create.url.query == b"uploads="
+ assert create.headers["content-type"] == "application/zip"
+ assert [request.url.query for request in uploads] == [
+ f"partNumber={number}&uploadId=up%2B1%2F%3D".encode() for number in (1, 2, 3)
+ ]
+ assert complete.url.query == b"uploadId=up%2B1%2F%3D"
+ assert complete.content == (
+ b""
+ b'1"etag-1"'
+ b'2"etag-2"'
+ b'3"etag-3"'
+ b""
+ )
+
+
+def test_put_file_retries_a_failed_part_on_its_own(tmp_path):
+ fake = FakeR2(fail_part={2: [503, 500]})
+
+ with fake.client() as client:
+ client.put_file("ledger-raw", "key", _file(tmp_path, b"abcdefghijkl"))
+
+ part_numbers = [
+ request.url.params["partNumber"]
+ for request in fake.requests
+ if request.method == "PUT"
+ ]
+ assert part_numbers == ["1", "2", "2", "2", "3"]
+ assert fake.parts == {1: b"abcde", 2: b"fghij", 3: b"kl"}
+
+
+@pytest.mark.parametrize("statuses", [[403], [503, 503, 503, 503]])
+def test_put_file_aborts_the_upload_when_a_part_fails(tmp_path, statuses):
+ fake = FakeR2(fail_part={2: statuses})
+
+ with fake.client() as client, pytest.raises(R2S3Error, match="HTTP"):
+ client.put_file("ledger-raw", "key", _file(tmp_path, b"abcdefghijkl"))
+
+ assert [request.method for request in fake.requests][-1] == "DELETE"
+ assert fake.requests[-1].url.query == b"uploadId=up%2B1%2F%3D"
+ assert 3 not in fake.parts
+ assert not any(
+ request.method == "POST" and "uploadId" in request.url.params
+ for request in fake.requests
+ )
+
+
+def test_put_file_fails_on_an_error_inside_a_successful_completion(tmp_path):
+ fake = FakeR2()
+ fake.complete_body = b"InternalError"
+
+ with fake.client() as client, pytest.raises(R2S3Error, match="InternalError"):
+ client.put_file("ledger-raw", "key", _file(tmp_path, b"abcdefghijkl"))
+
+ assert fake.requests[-1].method == "DELETE"
+
+
+def test_put_file_frees_each_part_before_reading_the_next(tmp_path):
+ # httpx keeps each response in a reference cycle with its request, which
+ # holds the part it sent. With the cycle collector off, every part would
+ # stay in memory unless put_file frees it.
+ part_size = 1 << 20
+ sizes: list[int] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "PUT":
+ sizes.append(len(request.content))
+ return httpx.Response(200, headers={"ETag": '"etag"'})
+ if "uploads" in request.url.params:
+ return httpx.Response(200, content=b"up")
+ return httpx.Response(200, content=b"")
+
+ path = _file(tmp_path, bytes(8 * part_size))
+ client = R2Client(
+ CREDENTIALS, transport=httpx.MockTransport(handler), part_size=part_size
+ )
+ gc.disable()
+ tracemalloc.start()
+ try:
+ with client:
+ client.put_file("ledger-raw", "key", path)
+ _, peak = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+ gc.enable()
+
+ assert sizes == [part_size] * 8
+ assert peak < 3 * part_size
+
+
+def test_object_size_reads_head_and_is_none_for_a_missing_key():
+ fake = FakeR2()
+ fake.objects["raw/uk/ucl/pkg/bundle_2.zip"] = 4_426_805_029
+
+ with fake.client() as client:
+ present = client.object_size("ledger-raw", "raw/uk/ucl/pkg/bundle_2.zip")
+ missing = client.object_size("ledger-raw", "raw/uk/ucl/pkg/bundle_3.zip")
+
+ assert present == 4_426_805_029
+ assert missing is None
+ assert [request.method for request in fake.requests] == ["HEAD", "HEAD"]
+
+
+def test_object_size_reports_a_refused_head_rather_than_a_missing_object():
+ fake = FakeR2()
+ fake.head_status = 403
+
+ with fake.client() as client, pytest.raises(R2S3Error, match="HTTP 403"):
+ client.object_size("ledger-raw", "key")
+
+
+def test_put_file_refuses_an_empty_file(tmp_path):
+ fake = FakeR2()
+
+ with fake.client() as client, pytest.raises(R2S3Error, match="non-empty"):
+ client.put_file("ledger-raw", "key", _file(tmp_path, b""))
+
+ assert fake.requests == []
+
+
+def test_explicit_r2_keys_need_no_token_lookup(tmp_path):
+ credentials = credentials_from_environment(
+ {
+ "CLOUDFLARE_ACCOUNT_ID": "account",
+ "R2_ACCESS_KEY_ID": "key-id",
+ "R2_SECRET_ACCESS_KEY": "secret",
+ "CLOUDFLARE_API_TOKEN": "unused",
+ },
+ wrangler_toml=tmp_path / "absent.toml",
+ transport=httpx.MockTransport(
+ lambda request: pytest.fail("no Cloudflare API call expected")
+ ),
+ )
+
+ assert credentials == R2Credentials("account", "key-id", "secret")
+ assert credentials.host == "account.r2.cloudflarestorage.com"
+ assert "secret" not in repr(credentials)
+
+
+@pytest.mark.parametrize("user_status", [200, 401])
+def test_api_token_gives_the_key_pair_and_wrangler_toml_the_account(
+ tmp_path, user_status
+):
+ wrangler_toml = tmp_path / "wrangler.toml"
+ wrangler_toml.write_text('name = "chronicle"\naccount_id = "acct-123"\n')
+ seen = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request.url.path)
+ assert request.headers["authorization"] == "Bearer token-value"
+ if request.url.path == "/client/v4/user/tokens/verify" and user_status != 200:
+ return httpx.Response(user_status, json={"success": False})
+ return httpx.Response(200, json={"result": {"id": "token-id"}})
+
+ credentials = credentials_from_environment(
+ {"CLOUDFLARE_API_TOKEN": "token-value"},
+ wrangler_toml=wrangler_toml,
+ transport=httpx.MockTransport(handler),
+ )
+
+ assert credentials == R2Credentials(
+ "acct-123", "token-id", hashlib.sha256(b"token-value").hexdigest()
+ )
+ expected_paths = ["/client/v4/user/tokens/verify"]
+ if user_status != 200:
+ expected_paths.append("/client/v4/accounts/acct-123/tokens/verify")
+ assert seen == expected_paths
+
+
+def test_a_token_lookup_that_cannot_reach_cloudflare_is_reported(tmp_path):
+ def unreachable(request: httpx.Request) -> httpx.Response:
+ raise httpx.ConnectError("offline", request=request)
+
+ with pytest.raises(R2S3Error, match="Could not verify"):
+ credentials_from_environment(
+ {"CLOUDFLARE_ACCOUNT_ID": "account", "CLOUDFLARE_API_TOKEN": "token"},
+ wrangler_toml=tmp_path / "absent.toml",
+ transport=httpx.MockTransport(unreachable),
+ )
+
+
+def test_an_unreadable_create_response_is_an_r2_error(tmp_path):
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(200, content=b"not xml")
+
+ client = R2Client(CREDENTIALS, transport=httpx.MockTransport(handler))
+ with client, pytest.raises(R2S3Error, match="no UploadId"):
+ client.put_file("ledger-raw", "key", _file(tmp_path, b"abc"))
+
+
+@pytest.mark.parametrize(
+ ("environ", "message"),
+ [
+ ({"CLOUDFLARE_API_TOKEN": "token"}, "No Cloudflare account"),
+ ({"CLOUDFLARE_ACCOUNT_ID": "account"}, "No R2 credentials"),
+ ],
+)
+def test_missing_credentials_are_reported(tmp_path, environ, message):
+ with pytest.raises(R2S3Error, match=message):
+ credentials_from_environment(environ, wrangler_toml=tmp_path / "absent.toml")
diff --git a/tests/test_source_ucl_house_price_per_square_metre_856542.py b/tests/test_source_ucl_house_price_per_square_metre_856542.py
new file mode 100644
index 00000000..bc524b38
--- /dev/null
+++ b/tests/test_source_ucl_house_price_per_square_metre_856542.py
@@ -0,0 +1,122 @@
+"""UCL's house price per square metre deposit, registered raw-only (chronicle#408, item 6).
+
+UK Data Service ReShare 856542 publishes two stored zips, of 3.2 and 4.4 GB, under CC BY
+4.0. Both carry Royal Mail address data licensed for non-commercial use. They are pinned and
+held in R2 only, uploaded through R2's S3 API because Wrangler takes at most 300 MiB
+(chronicle#409), and no build parses them.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+
+import yaml
+
+from chronicle.artifacts import (
+ WRANGLER_PUT_LIMIT_BYTES,
+ build_r2_key,
+ inventory_source_artifacts,
+ publish_source_artifacts,
+)
+from chronicle.bundle import UK_BUNDLE_SOURCES
+from chronicle.source_package import SOURCE_PACKAGE_ALIASES
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+DIRECTORY = "ucl/house_price_per_square_metre_856542"
+PACKAGE_DIR = REPOSITORY_ROOT / "db" / "data" / DIRECTORY
+PACKAGE_ID = "ucl-house-price-per-square-metre-856542"
+# file key -> filename, sha256, bytes, and ReShare's ETag, which is the file's MD5.
+BUNDLES = {
+ "bundle_1": (
+ "856542_bundle_1.zip",
+ "e397cdac214b918a39f3ccf5266adaf9e4c3d8fb4f71e0bdc1b9c9474734b296",
+ 3_214_161_126,
+ "239aebb08e10939b7cba4c5b2d5f89c9",
+ ),
+ "bundle_2": (
+ "856542_bundle_2.zip",
+ "61cbf6a72e35f7569d9121d80eaf89e757fec3bd4c21f1175e37fff44e02da97",
+ 4_426_805_029,
+ "5fa5f769f3061aef86c455a4ed23c97e",
+ ),
+}
+
+
+def _manifest() -> dict:
+ return yaml.safe_load((PACKAGE_DIR / "manifest.yaml").read_text(encoding="utf-8"))
+
+
+def test_both_bundles_are_pinned_raw_only_in_r2():
+ manifest = _manifest()
+ assert manifest["source_id"] == "ucl"
+ assert manifest["package_id"] == PACKAGE_ID
+ assert manifest["registration_kind"] == "raw_only"
+ assert manifest["license"] == (
+ "Creative Commons Attribution 4.0 International (CC BY 4.0)"
+ )
+ assert manifest["license_verified_at"] == "2026-11-06"
+ assert "non-commercial" in manifest["license_evidence"]
+ assert "10.5255/UKDA-SN-856542" in manifest["attribution"]
+ assert list(manifest["files"]) == list(BUNDLES)
+ for file_key, (filename, sha256, size_bytes, md5) in BUNDLES.items():
+ entry = manifest["files"][file_key]
+ number = file_key.removeprefix("bundle_")
+ assert entry["filename"] == filename
+ assert entry["source_url"] == (
+ f"https://reshare.ukdataservice.ac.uk/856542/{number}/{filename}"
+ )
+ assert entry["sha256"] == sha256
+ assert entry["size_bytes"] == size_bytes
+ assert md5 in manifest["provenance_notes"]
+ # Too large for Wrangler, so publish-raw sends it through the S3 API.
+ assert size_bytes > WRANGLER_PUT_LIMIT_BYTES
+
+ key = build_r2_key(
+ source_id="ucl",
+ package_id=PACKAGE_ID,
+ year=file_key,
+ sha256=sha256,
+ filename=filename,
+ package_path=PACKAGE_DIR / "manifest.yaml",
+ )
+ assert key == f"raw/uk/ucl/{PACKAGE_ID}/{file_key}/{sha256}/{filename}"
+ assert entry["storage"] == {
+ "local": False,
+ "r2": {
+ "provider": "r2",
+ "bucket": "ledger-raw",
+ "key": key,
+ "uri": f"r2://ledger-raw/{key}",
+ },
+ }
+ ignored = subprocess.run(
+ ["git", "check-ignore", "--quiet", str(PACKAGE_DIR / filename)],
+ cwd=REPOSITORY_ROOT,
+ check=False,
+ )
+ assert ignored.returncode == 0
+
+ assert PACKAGE_ID not in SOURCE_PACKAGE_ALIASES
+ assert PACKAGE_ID not in UK_BUNDLE_SOURCES
+ assert not (
+ REPOSITORY_ROOT / "packages" / DIRECTORY / "source_package.yaml"
+ ).exists()
+
+
+def test_a_checkout_without_the_bundles_stays_valid(tmp_path):
+ target = tmp_path / "db" / "data" / DIRECTORY
+ target.mkdir(parents=True)
+ (target / "manifest.yaml").write_bytes((PACKAGE_DIR / "manifest.yaml").read_bytes())
+
+ inventory = inventory_source_artifacts(tmp_path / "db" / "data")
+ published = publish_source_artifacts(
+ tmp_path / "db" / "data", wrangler_command="false"
+ )
+
+ assert inventory.valid
+ assert inventory.counts["missing_count"] == 0
+ assert inventory.counts["r2_only_count"] == 2
+ assert published.valid
+ assert published.counts["skipped_count"] == 2
+ assert published.counts["uploaded_count"] == 0