From d954c685390d39818ae1a99befeba64756a72a40 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 10:24:08 +0800 Subject: [PATCH 1/4] feat: add M8 verified research inputs --- README.md | 35 +- pyproject.toml | 3 + src/quant_data_kit/__init__.py | 38 + src/quant_data_kit/_version.py | 2 +- src/quant_data_kit/curated.py | 476 ++++++++- src/quant_data_kit/research_contracts_v2.py | 433 ++++++++ src/quant_data_kit/research_inputs_v2.py | 963 +++++++++++++++++ tests/test_m2_integration.py | 7 +- tests/test_m8_research_contracts.py | 237 +++++ tests/test_m8_research_inputs.py | 1033 +++++++++++++++++++ tools/check_branch_coverage.py | 2 + 11 files changed, 3222 insertions(+), 7 deletions(-) create mode 100644 src/quant_data_kit/research_contracts_v2.py create mode 100644 src/quant_data_kit/research_inputs_v2.py create mode 100644 tests/test_m8_research_contracts.py create mode 100644 tests/test_m8_research_inputs.py diff --git a/README.md b/README.md index 719bc12..bdc6a35 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ move or rebuild a release tag to repair a dependency resolution. | `temporal_v2` | Strict bitemporal validation and PIT joins without silent fallback | | `data_lake` | Immutable Raw bytes, strict partitioned Normalized Parquet, quarantine, pinned DuckDB reads and storage stop policy | | `curated` | Session-aware bar aggregation and immutable revision/lineage snapshots | +| `research_contracts_v2` | Closed M8 Curated aggregation and verified-factor-input contracts | +| `research_inputs_v2` | Content-addressed market context plus fail-closed Curated/Normalized factor-input factories | | `l2_replay` | Deterministic Snapshot+Delta reconstruction, sequence/cross checks and checkpoint hashes | | `adapters_v2` | Binance, OKX and supplier-neutral domestic desensitized fixture adapters | | `capture_v2` | Fail-closed public Binance/OKX L2 capture, immutable batched Raw segments, snapshot synchronization, independent archive/restore verification and the Raw-to-Normalized bridge | @@ -86,11 +88,42 @@ The public-feed collector's exact eight-stream scope, explicit storage configura safe CLI modes and non-certification boundary are documented in [`docs/m7-crypto-l2-capture.md`](docs/m7-crypto-l2-capture.md). +## M8 certified research inputs + +M8 factor code must consume one of two public factories instead of promoting an arbitrary Arrow +table or the ordinary `read_normalized_events` result: + +```python +from quant_data_kit import ( + EventSchemaRef, + create_market_context_snapshot, + load_verified_curated_bars, + load_verified_normalized_events, +) + +bars = load_verified_curated_bars(root, "bars-1m", curated_snapshot_id) +events = load_verified_normalized_events( + root, + normalized_snapshot_id, + [EventSchemaRef("puresaber.trade-event", "2.0.0")], + market_context_snapshot_id, +) +``` + +`create_market_context_snapshot` content-addresses immutable `InstrumentSpec` and +`TradingSession` values together with one explicit calendar and session-policy version. A certified +Curated snapshot additionally binds `puresaber.curated-aggregation@1.0.0`: fixed interval, +session/trading-day rollup, or event-Bar threshold and exact source-range evidence. The loaders +verify the complete source snapshot, physical and logical partition hashes, PIT timestamps, +context membership, ordering, L2 snapshot/delta replay, selection hash and a second post-read +snapshot check. Legacy Curated manifests remain readable through `load_curated_snapshot` but fail +with `legacy-curated-not-m8-certified` at the certified factory. + ## M2 data-lake guarantees - Raw persists exact provider bytes through lake-local staging and atomic rename. Its integrity anchor binds source, request, UTC collection time, object/key path identity, SHA-256 and the30-day retention policy. A crash-released process lock and immutable key claim make concurrent writes, crash recovery and cleanup serialize on the same idempotency key. Every write, read and cleanup rejects path escapes and Windows reparse points below the lake root. - Normalized requires resolvable, hash-verified Raw references and writes only frozen`standard/v2`Arrow schemas under`provider/venue/event_type/date/instrument`partitions. Capture epochs persist`PREPARED`before snapshot publication and finish as`COMMITTED`or`ABORTED`; startup uses the frozen stream configuration as an independent identity anchor, enforces closed terminal JSON fields and strict types, recomputes partition rows, logical hashes, the available-time maximum and the final L2 state from the immutable journal, and rejects any receipt bound to a different snapshot before network startup. A sharded persistent claim index binds every lake-wide`event_id`to its Arrow-normalized logical event hash. Same-ID/same-content reuse is idempotent; conflicting content, bad sequences and L2 reconstruction failures cannot enter research snapshots. -- The certified Curated entry is`curate_trade_bars_from_snapshot`: it reads trades from one explicit verified Normalized snapshot, constructs session-aware bars and binds the exact lineage. One`dataset+revision_id`maps to one snapshot; corrected data requires a new revision and never overwrites history. +- Certified Curated producers are`curate_trade_bars_from_snapshot`、`curate_session_bars_from_snapshot`and`curate_trade_event_bars_from_snapshot`: they read trades from one explicit verified Normalized snapshot, construct authoritative fixed/session/event Bars and bind exact lineage plus an immutable market-context snapshot. One`dataset+revision_id`maps to one snapshot; corrected data requires a new revision and never overwrites history. - Normalized and Curated snapshot identities bind Arrow-canonical logical rows and physical Parquet hashes. DuckDB verifies the fixed snapshot, copies Arrow data into in-memory tables, then disables external access; user SQL cannot call file readers or resolve`latest`/`main`. - Collection stops with a visible`COLLECTION_STOPPED`error if hot data would exceed150GB or free space would fall below`max(volume*20%,100GB)`. - Raw cleanup requires all of: the30-day window elapsed, explicit confirmation, an accessible real local archive object, archive hash equality and a successful restore-hash exercise. Cleanup publishes an immutable audit tombstone and resumes an explicit`deleting`state after interruption; local unlink failures remain visible to callers. Remote archives have no M2 verifier and therefore stop cleanup. No background or silent deletion path exists. diff --git a/pyproject.toml b/pyproject.toml index 611d276..fbf5779 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,5 +51,8 @@ layer = "data" schemas = [ { id = "puresaber.market-events", version = "2.0.0" }, { id = "puresaber.instrument-master", version = "2.0.0" }, + { id = "puresaber.curated-aggregation", version = "1.0.0" }, + { id = "puresaber.market-context", version = "1.0.0" }, + { id = "puresaber.verified-factor-input", version = "1.0.0" }, ] lock-files = ["requirements.lock"] diff --git a/src/quant_data_kit/__init__.py b/src/quant_data_kit/__init__.py index 73aa842..68c87dd 100644 --- a/src/quant_data_kit/__init__.py +++ b/src/quant_data_kit/__init__.py @@ -28,8 +28,12 @@ from quant_data_kit.curated import ( CuratedSnapshot, Curator, + build_event_bars, build_session_bars, + build_session_rollup_bars, + curate_session_bars_from_snapshot, curate_trade_bars_from_snapshot, + curate_trade_event_bars_from_snapshot, load_curated_snapshot, ) from quant_data_kit.data_lake import ( @@ -93,6 +97,23 @@ merge_earnings_to_panel, merge_northbound_to_panel, ) +from quant_data_kit.research_contracts_v2 import ( + CURATED_AGGREGATION_SCHEMA_ID, + MARKET_CONTEXT_SCHEMA_ID, + VERIFIED_FACTOR_INPUT_SCHEMA_ID, + CuratedAggregation, + EventBarPartitionEvidence, + EventSchemaRef, + LineageRef, + VerifiedFactorInput, +) +from quant_data_kit.research_inputs_v2 import ( + MarketContextSnapshot, + create_market_context_snapshot, + load_market_context_snapshot, + load_verified_curated_bars, + load_verified_normalized_events, +) from quant_data_kit.schemas_v2 import ( SCHEMA_VERSION_V2, get_arrow_schema, @@ -124,9 +145,12 @@ from quant_data_kit.validate import validate_price_frame __all__ = [ + "CURATED_AGGREGATION_SCHEMA_ID", "M7_CAPABILITIES", "M7_PROVIDERS", + "MARKET_CONTEXT_SCHEMA_ID", "SCHEMA_VERSION_V2", + "VERIFIED_FACTOR_INPUT_SCHEMA_ID", "AdapterContext", "AdapterInstrument", "AggressorSide", @@ -150,21 +174,26 @@ "CollectionStoppedError", "CorporateActionEvent", "CryptoL2CaptureCoordinator", + "CuratedAggregation", "CuratedSnapshot", "Curator", "DataManifest", "DatasetSnapshot", "DuckDBCatalog", + "EventBarPartitionEvidence", + "EventSchemaRef", "FixedPoint", "FundingRateEvent", "InstrumentSpec", "L2BookReconstructor", "L2ReplayError", "L2ReplayResult", + "LineageRef", "LocalArchiveController", "MarginMode", "MarkPriceEvent", "MarketClock", + "MarketContextSnapshot", "MarketEvent", "NormalizationResult", "NormalizedSnapshot", @@ -182,16 +211,22 @@ "TemporalAudit", "TradeEvent", "TradingSession", + "VerifiedFactorInput", "VolumeIdentity", "__version__", "adapt_fixture_messages", "add_industry_relative_strength", "audit_point_in_time", + "build_event_bars", "build_session_bars", + "build_session_rollup_bars", "cache_covers_range", "cleanup_archived_raw_object", + "create_market_context_snapshot", "create_snapshot", + "curate_session_bars_from_snapshot", "curate_trade_bars_from_snapshot", + "curate_trade_event_bars_from_snapshot", "dataclass_payload", "default_crypto_l2_streams", "ensure_utc_datetime", @@ -201,10 +236,13 @@ "incremental_start_date", "load_curated_snapshot", "load_manifest", + "load_market_context_snapshot", "load_normalized_snapshot", "load_parquet", "load_raw_object", "load_snapshot", + "load_verified_curated_bars", + "load_verified_normalized_events", "market_event_payload", "merge_earnings_to_panel", "merge_northbound_to_panel", diff --git a/src/quant_data_kit/_version.py b/src/quant_data_kit/_version.py index 8821bf5..0cc858e 100644 --- a/src/quant_data_kit/_version.py +++ b/src/quant_data_kit/_version.py @@ -1,3 +1,3 @@ """Single authoritative package version used by builds and runtime imports.""" -__version__ = "0.7.4" +__version__ = "0.8.0" diff --git a/src/quant_data_kit/curated.py b/src/quant_data_kit/curated.py index d67f802..631a21a 100644 --- a/src/quant_data_kit/curated.py +++ b/src/quant_data_kit/curated.py @@ -32,6 +32,11 @@ from quant_data_kit.exceptions import ValidationError from quant_data_kit.fixed_point import FixedPoint from quant_data_kit.market_events_v2 import BarEvent, market_event_payload +from quant_data_kit.research_contracts_v2 import ( + CuratedAggregation, + EventBarPartitionEvidence, + EventSchemaRef, +) from quant_data_kit.schemas_v2 import ( BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2, @@ -72,6 +77,7 @@ class CuratedSnapshot: rows: int lineage: dict[str, str] partitions: tuple[CuratedPartition, ...] + aggregation: CuratedAggregation | None = None def _canonical(value: Any) -> bytes: @@ -196,7 +202,10 @@ def build_session_bars( for (instrument_id, trading_day, session_id, upstream_source, bar_start), trades in sorted( grouped.items(), key=lambda item: item[0] ): - ordered = sorted(trades, key=lambda item: (item["event_time"], item["event_id"])) + ordered = sorted( + trades, + key=lambda item: (item["event_time"], int(item["sequence"]), item["event_id"]), + ) price_scale = max(int(item["price"]["scale"]) for item in ordered) quantity_scale = max(int(item["quantity"]["scale"]) for item in ordered) prices = [_fixed_decimal(item["price"]) for item in ordered] @@ -244,6 +253,222 @@ def build_session_bars( return bars +def _bar_from_trade_group( + trades: list[dict[str, Any]], + *, + bar_start: datetime, + bar_end: datetime, + source: str, + recipe_version: str, + identity_extra: Mapping[str, Any], + output_session_id: str | None = None, +) -> dict[str, Any]: + if not trades: + raise ValidationError("Cannot build a Bar from no trades") + ordered = sorted( + trades, + key=lambda item: ( + _utc(str(item["event_time"]), "event_time"), + int(item["sequence"]), + str(item["event_id"]), + ), + ) + if not bar_start < bar_end: + raise ValidationError("Bar boundaries must be strictly positive") + price_scale = max(int(item["price"]["scale"]) for item in ordered) + quantity_scale = max(int(item["quantity"]["scale"]) for item in ordered) + prices = [_fixed_decimal(item["price"]) for item in ordered] + volume = sum((_fixed_decimal(item["quantity"]) for item in ordered), Decimal(0)) + received_at = max( + bar_end, + max(_utc(str(item["received_at"]), "received_at") for item in ordered), + ) + available_at = max( + received_at, + max(_utc(str(item["available_at"]), "available_at") for item in ordered), + ) + identity = { + "recipe_version": recipe_version, + "upstream_event_ids": [item["event_id"] for item in ordered], + "bar_start": _utc_text(bar_start), + "bar_end": _utc_text(bar_end), + **dict(identity_extra), + } + event = BarEvent( + event_id=f"bar-{_hash_bytes(_canonical(identity))[:24]}", + instrument_id=str(ordered[0]["instrument_id"]), + event_time=bar_end, + received_at=received_at, + available_at=available_at, + source=source, + trading_day=datetime.fromisoformat(str(ordered[0]["trading_day"])).date(), + session_id=output_session_id or str(ordered[-1]["session_id"]), + sequence=int(ordered[-1]["sequence"]), + bar_start=bar_start, + bar_end=bar_end, + open_price=_fixed(prices[0], price_scale), + high_price=_fixed(max(prices), price_scale), + low_price=_fixed(min(prices), price_scale), + close_price=_fixed(prices[-1], price_scale), + volume=_fixed(volume, quantity_scale), + is_complete=True, + ) + payload = market_event_payload(event) + payload["source"] = source + validate_json_record(BAR_EVENT_SCHEMA_ID, payload) + return payload + + +def build_session_rollup_bars( + records: Iterable[Mapping[str, Any]], + *, + session_boundaries: Mapping[str, tuple[datetime, datetime]], + session_rollup: str, + trading_day_boundaries: Mapping[tuple[str, str], tuple[datetime, datetime, str]] | None = None, + instrument_venues: Mapping[str, str] | None = None, + source: str = "curated", + recipe_version: str = "session-rollup-v1", +) -> list[dict[str, Any]]: + """Aggregate complete session or trading-day Bars from frozen trade events.""" + if session_rollup not in {"session", "trading_day"}: + raise ValidationError("session_rollup must be session or trading_day") + grouped: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) + for raw in records: + record = dict(raw) + validate_json_record(TRADE_EVENT_SCHEMA_ID, record) + session_id = str(record["session_id"]) + if session_id not in session_boundaries: + raise ValidationError(f"Missing authoritative boundary for {session_id}") + key = ( + str(record["instrument_id"]), + str(record["trading_day"]), + str(record["source"]), + *([session_id] if session_rollup == "session" else []), + ) + grouped[key].append(record) + bars: list[dict[str, Any]] = [] + for key, trades in sorted(grouped.items()): + referenced = {str(item["session_id"]) for item in trades} + output_session_id: str | None = None + if session_rollup == "trading_day": + instrument_id, trading_day = key[:2] + venue = (instrument_venues or {}).get(instrument_id) + boundary = (trading_day_boundaries or {}).get((str(venue), trading_day)) + if venue is None or boundary is None: + raise ValidationError( + f"Missing authoritative trading-day boundary for {instrument_id}/{trading_day}" + ) + starts = [_utc(boundary[0], f"{trading_day}.opens_at")] + ends = [_utc(boundary[1], f"{trading_day}.closes_at")] + output_session_id = boundary[2] + else: + starts = [_utc(session_boundaries[item][0], f"{item}.opens_at") for item in referenced] + ends = [_utc(session_boundaries[item][1], f"{item}.closes_at") for item in referenced] + bars.append( + _bar_from_trade_group( + trades, + bar_start=min(starts), + bar_end=max(ends), + source=source, + recipe_version=recipe_version, + identity_extra={"session_rollup": session_rollup, "group": list(key)}, + output_session_id=output_session_id, + ) + ) + return bars + + +def build_event_bars( + records: Iterable[Mapping[str, Any]], + *, + basis: str, + threshold: FixedPoint, + session_starts: Mapping[str, datetime], + source: str = "curated", + recipe_version: str = "event-bars-v1", + require_complete: bool = True, +) -> list[dict[str, Any]]: + """Build deterministic trade-count, base-volume, or quote-notional Bars.""" + if basis not in {"trade_count", "base_volume", "quote_notional"}: + raise ValidationError("unsupported event-bar basis") + if not isinstance(threshold, FixedPoint) or not threshold.is_positive(): + raise ValidationError("event-bar threshold must be a positive FixedPoint") + if basis == "trade_count" and threshold.scale != 0: + raise ValidationError("trade_count threshold must have scale zero") + grouped: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) + for raw in records: + record = dict(raw) + validate_json_record(TRADE_EVENT_SCHEMA_ID, record) + session_id = str(record["session_id"]) + if session_id not in session_starts: + raise ValidationError(f"Missing session start for {session_id}") + grouped[ + ( + str(record["source"]), + str(record["instrument_id"]), + str(record["trading_day"]), + session_id, + ) + ].append(record) + + threshold_value = threshold.to_decimal() + bars: list[dict[str, Any]] = [] + for stream, trades in sorted(grouped.items()): + ordered = sorted( + trades, + key=lambda item: ( + _utc(str(item["event_time"]), "event_time"), + int(item["sequence"]), + str(item["event_id"]), + ), + ) + prior_identity: tuple[datetime, int, str] | None = None + bar_start = _utc(session_starts[stream[3]], f"{stream[3]}.opens_at") + bucket: list[dict[str, Any]] = [] + accumulated = Decimal(0) + for trade in ordered: + identity = ( + _utc(str(trade["event_time"]), "event_time"), + int(trade["sequence"]), + str(trade["event_id"]), + ) + if prior_identity is not None and identity <= prior_identity: + raise ValidationError(f"event-bar source stream is not strictly ordered: {stream}") + prior_identity = identity + bucket.append(trade) + if basis == "trade_count": + accumulated += 1 + elif basis == "base_volume": + accumulated += _fixed_decimal(trade["quantity"]) + else: + accumulated += _fixed_decimal(trade["price"]) * _fixed_decimal(trade["quantity"]) + if accumulated < threshold_value: + continue + bar_end = identity[0] + bars.append( + _bar_from_trade_group( + bucket, + bar_start=bar_start, + bar_end=bar_end, + source=source, + recipe_version=recipe_version, + identity_extra={ + "event_bar_basis": basis, + "event_bar_threshold": { + "units": str(threshold.units), + "scale": threshold.scale, + }, + }, + ) + ) + bar_start = bar_end + bucket = [] + accumulated = Decimal(0) + if bucket and require_complete: + raise ValidationError(f"event-bar source stream ends below threshold: {stream}") + return bars + + def _arrow_ready_bar(record: Mapping[str, Any]) -> dict[str, Any]: result = dict(record) schema = get_arrow_schema(BAR_EVENT_SCHEMA_ID) @@ -267,8 +492,9 @@ def _snapshot_identity( created_at: str, lineage: Mapping[str, str], partitions: tuple[CuratedPartition, ...], + aggregation: CuratedAggregation | None = None, ) -> dict[str, Any]: - return { + identity = { "schema_version": SCHEMA_VERSION_V2, "layer": "curated", "dataset": dataset, @@ -278,6 +504,26 @@ def _snapshot_identity( "lineage": dict(sorted(lineage.items())), "partitions": [asdict(item) for item in partitions], } + if aggregation is not None: + identity["aggregation"] = aggregation.to_contract() + return identity + + +def _snapshot_manifest(snapshot: CuratedSnapshot) -> dict[str, Any]: + return { + "schema_version": snapshot.schema_version, + "layer": snapshot.layer, + "dataset": snapshot.dataset, + "snapshot_id": snapshot.snapshot_id, + "revision_id": snapshot.revision_id, + "recipe_version": snapshot.recipe_version, + "created_at": snapshot.created_at, + "logical_sha256": snapshot.logical_sha256, + "rows": snapshot.rows, + "lineage": dict(snapshot.lineage), + "partitions": [asdict(item) for item in snapshot.partitions], + "aggregation": snapshot.aggregation.to_contract() if snapshot.aggregation else None, + } def _revision_record(snapshot: CuratedSnapshot) -> dict[str, str]: @@ -407,6 +653,7 @@ def _write_curated_bars( recipe_version: str, normalized_snapshot_id: str, policy: StoragePolicy, + aggregation: CuratedAggregation | None = None, ) -> CuratedSnapshot: """Persist bars only after loading their exact immutable Normalized lineage.""" lake_root = _resolved_lake_root(root, create=False) @@ -414,6 +661,8 @@ def _write_curated_bars( revision_id = _revision_segment(revision_id) if not recipe_version.strip(): raise ValidationError("recipe_version is required") + if aggregation is not None and aggregation.recipe_version != recipe_version: + raise ValidationError("aggregation recipe_version must match the Curated recipe") normalized = load_normalized_snapshot(lake_root, normalized_snapshot_id) lineage = { "normalized_snapshot_id": normalized.snapshot_id, @@ -444,7 +693,10 @@ def _write_curated_bars( policy=policy, ) for (trading_date, instrument_id), group in sorted(groups.items()): - ordered = sorted(group, key=lambda row: (row["event_time"], row["event_id"])) + ordered = sorted( + group, + key=lambda row: (row["event_time"], int(row["sequence"]), row["event_id"]), + ) table = pa.Table.from_pylist( [_arrow_ready_bar(record) for record in ordered], schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID), @@ -479,6 +731,7 @@ def _write_curated_bars( created_at=created_at, lineage=lineage, partitions=partitions, + aggregation=aggregation, ) logical_sha256 = _hash_bytes(_canonical(identity)) snapshot_id = f"sha256-{logical_sha256}" @@ -494,9 +747,10 @@ def _write_curated_bars( rows=sum(item.rows for item in partitions), lineage=dict(sorted(lineage.items())), partitions=partitions, + aggregation=aggregation, ) (stage / "manifest.json").write_text( - json.dumps(asdict(snapshot), indent=2, ensure_ascii=False), + json.dumps(_snapshot_manifest(snapshot), indent=2, ensure_ascii=False), encoding="utf-8", ) return _publish_curated_snapshot( @@ -517,6 +771,7 @@ def curate_trade_bars_from_snapshot( interval: timedelta, session_starts: Mapping[str, datetime], source: str = "curated", + market_context_snapshot_id: str | None = None, policy: StoragePolicy | None = None, ) -> CuratedSnapshot: """Certified public path: fixed Normalized trade snapshot to immutable Curated bars.""" @@ -532,6 +787,33 @@ def curate_trade_bars_from_snapshot( source=source, recipe_version=recipe_version, ) + aggregation: CuratedAggregation | None = None + if market_context_snapshot_id is not None: + from quant_data_kit.research_inputs_v2 import load_market_context_snapshot + + context = load_market_context_snapshot(root, market_context_snapshot_id) + authoritative_starts = {item.session_id: item.opens_at for item in context.sessions} + used_session_ids = {str(item["session_id"]) for item in trades} + for session_id in used_session_ids: + if session_id not in session_starts or session_id not in authoritative_starts: + raise ValidationError(f"Missing market-context session start for {session_id}") + if _utc(session_starts[session_id], session_id) != authoritative_starts[session_id]: + raise ValidationError( + f"session_starts differs from market context for {session_id}" + ) + interval_us = ( + interval.days * 86_400 + interval.seconds + ) * 1_000_000 + interval.microseconds + aggregation = CuratedAggregation( + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + kind="fixed_time_bar", + recipe_version=recipe_version, + interval_ns=interval_us * 1_000, + market_context_snapshot_id=context.snapshot_id, + market_context_logical_sha256=context.logical_sha256, + source_event_schemas=(EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + ) return _write_curated_bars( root, bars, @@ -540,6 +822,187 @@ def curate_trade_bars_from_snapshot( recipe_version=recipe_version, normalized_snapshot_id=normalized.snapshot_id, policy=resolved_policy, + aggregation=aggregation, + ) + + +def curate_session_bars_from_snapshot( + root: Path, + *, + normalized_snapshot_id: str, + dataset: str, + revision_id: str, + recipe_version: str, + session_rollup: str, + market_context_snapshot_id: str, + source: str = "curated", + policy: StoragePolicy | None = None, +) -> CuratedSnapshot: + """Create one complete Bar per authoritative session or trading day.""" + from quant_data_kit.research_inputs_v2 import load_market_context_snapshot + + normalized = load_normalized_snapshot(root, normalized_snapshot_id) + context = load_market_context_snapshot(root, market_context_snapshot_id) + trades = read_normalized_events(root, normalized.snapshot_id, event_type="trade") + if not trades: + raise ValidationError("Normalized snapshot contains no trades to curate") + boundaries = {item.session_id: (item.opens_at, item.closes_at) for item in context.sessions} + sessions_by_venue_day: dict[tuple[str, str], list[Any]] = defaultdict(list) + for session in context.sessions: + sessions_by_venue_day[(session.venue, session.trading_day.isoformat())].append(session) + day_boundaries = { + key: ( + min(item.opens_at for item in values), + max(item.closes_at for item in values), + max(values, key=lambda item: item.closes_at).session_id, + ) + for key, values in sessions_by_venue_day.items() + } + instrument_venues = {item.instrument_id: item.venue for item in context.instruments} + bars = build_session_rollup_bars( + trades, + session_boundaries=boundaries, + session_rollup=session_rollup, + trading_day_boundaries=day_boundaries, + instrument_venues=instrument_venues, + source=source, + recipe_version=recipe_version, + ) + aggregation = CuratedAggregation( + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + kind="session_bar", + recipe_version=recipe_version, + session_rollup=session_rollup, + market_context_snapshot_id=context.snapshot_id, + market_context_logical_sha256=context.logical_sha256, + source_event_schemas=(EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + ) + return _write_curated_bars( + root, + bars, + dataset=dataset, + revision_id=revision_id, + recipe_version=recipe_version, + normalized_snapshot_id=normalized.snapshot_id, + policy=policy or StoragePolicy(), + aggregation=aggregation, + ) + + +def _source_selection_sha256(records: list[dict[str, Any]]) -> str: + payload = { + "algorithm": "puresaber.event-selection-canonical-json@1.0.0", + "schema_id": TRADE_EVENT_SCHEMA_ID, + "schema_version": SCHEMA_VERSION_V2, + "records": records, + } + return _hash_bytes(_canonical(payload)) + + +def curate_trade_event_bars_from_snapshot( + root: Path, + *, + normalized_snapshot_id: str, + dataset: str, + revision_id: str, + recipe_version: str, + basis: str, + threshold: FixedPoint, + market_context_snapshot_id: str, + source: str = "curated", + policy: StoragePolicy | None = None, +) -> CuratedSnapshot: + """Create certified event Bars with source-range evidence for every stream.""" + from quant_data_kit.research_inputs_v2 import load_market_context_snapshot + + normalized = load_normalized_snapshot(root, normalized_snapshot_id) + context = load_market_context_snapshot(root, market_context_snapshot_id) + trades = [ + dict(item) + for item in read_normalized_events(root, normalized.snapshot_id, event_type="trade") + ] + if not trades: + raise ValidationError("Normalized snapshot contains no trades to curate") + session_starts = {item.session_id: item.opens_at for item in context.sessions} + bars = build_event_bars( + trades, + basis=basis, + threshold=threshold, + session_starts=session_starts, + source=source, + recipe_version=recipe_version, + require_complete=True, + ) + grouped: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) + for record in trades: + grouped[ + ( + str(record["source"]), + str(record["instrument_id"]), + str(record["trading_day"]), + str(record["session_id"]), + ) + ].append(record) + evidence: list[EventBarPartitionEvidence] = [] + for (upstream_source, instrument_id, trading_day, session_id), rows in sorted(grouped.items()): + ordered = sorted( + rows, + key=lambda item: ( + _utc(str(item["event_time"]), "event_time"), + int(item["sequence"]), + str(item["event_id"]), + ), + ) + relative_path = Path( + f"date={trading_day}/instrument={quote(instrument_id, safe='-._')}/data.parquet" + ).as_posix() + evidence.append( + EventBarPartitionEvidence( + relative_path=relative_path, + source=upstream_source, + instrument_id=instrument_id, + session_id=session_id, + first_sequence=int(ordered[0]["sequence"]), + last_sequence=int(ordered[-1]["sequence"]), + first_event_id=str(ordered[0]["event_id"]), + last_event_id=str(ordered[-1]["event_id"]), + event_count=len(ordered), + source_selection_sha256=_source_selection_sha256(ordered), + ) + ) + ordered_evidence = tuple( + sorted( + evidence, + key=lambda item: ( + item.stream_key, + item.first_sequence, + item.last_sequence, + item.relative_path, + ), + ) + ) + aggregation = CuratedAggregation( + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + kind="event_bar", + recipe_version=recipe_version, + event_bar_basis=basis, + event_bar_threshold=threshold, + market_context_snapshot_id=context.snapshot_id, + market_context_logical_sha256=context.logical_sha256, + source_event_schemas=(EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + partition_evidence=ordered_evidence, + ) + return _write_curated_bars( + root, + bars, + dataset=dataset, + revision_id=revision_id, + recipe_version=recipe_version, + normalized_snapshot_id=normalized.snapshot_id, + policy=policy or StoragePolicy(), + aggregation=aggregation, ) @@ -568,6 +1031,10 @@ def _load_curated_snapshot( raise ValidationError(f"Curated snapshot manifest missing: {manifest_path}") payload = json.loads(manifest_path.read_text(encoding="utf-8")) payload["partitions"] = tuple(CuratedPartition(**item) for item in payload["partitions"]) + raw_aggregation = payload.get("aggregation") + payload["aggregation"] = ( + CuratedAggregation.from_contract(raw_aggregation) if raw_aggregation is not None else None + ) snapshot = CuratedSnapshot(**payload) if ( snapshot.snapshot_id != snapshot_id @@ -595,6 +1062,7 @@ def _load_curated_snapshot( created_at=snapshot.created_at, lineage=snapshot.lineage, partitions=snapshot.partitions, + aggregation=snapshot.aggregation, ) logical_sha256 = _hash_bytes(_canonical(identity)) if logical_sha256 != snapshot.logical_sha256 or snapshot_id != f"sha256-{logical_sha256}": diff --git a/src/quant_data_kit/research_contracts_v2.py b/src/quant_data_kit/research_contracts_v2.py new file mode 100644 index 0000000..8ac034b --- /dev/null +++ b/src/quant_data_kit/research_contracts_v2.py @@ -0,0 +1,433 @@ +"""Closed M8 contracts shared by certified research-input factories.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +import pyarrow as pa + +from quant_data_kit.exceptions import ValidationError +from quant_data_kit.fixed_point import FixedPoint + +VERIFIED_FACTOR_INPUT_SCHEMA_ID = "puresaber.verified-factor-input@1.0.0" +CURATED_AGGREGATION_SCHEMA_ID = "puresaber.curated-aggregation@1.0.0" +MARKET_CONTEXT_SCHEMA_ID = "puresaber.market-context@1.0.0" + +_HASH = re.compile(r"^[0-9a-f]{64}$") +_SNAPSHOT_ID = re.compile(r"^sha256-[0-9a-f]{64}$") +_SCHEMA_ID = re.compile(r"^puresaber\.[a-z0-9._-]+$") +_SEMVER = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +_CANONICAL_NONNEGATIVE = re.compile(r"^(?:0|[1-9][0-9]*)$") +_CANONICAL_POSITIVE = re.compile(r"^[1-9][0-9]*$") +_INT64_MAX = 2**63 - 1 + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValidationError(f"{field_name} must be a non-empty string") + return value + + +def _require_hash(value: str, field_name: str) -> str: + if not isinstance(value, str) or _HASH.fullmatch(value) is None: + raise ValidationError(f"{field_name} must be a lowercase SHA-256") + return value + + +def _require_snapshot_id(value: str, field_name: str) -> str: + if not isinstance(value, str) or _SNAPSHOT_ID.fullmatch(value) is None: + raise ValidationError(f"{field_name} must be a content-addressed snapshot ID") + return value + + +def _closed_payload(payload: Mapping[str, Any], fields: set[str], name: str) -> None: + if not isinstance(payload, Mapping) or set(payload) != fields: + raise ValidationError(f"{name} must contain exactly {sorted(fields)}") + + +def _canonical_integer(value: Any, field_name: str, *, positive: bool) -> int: + pattern = _CANONICAL_POSITIVE if positive else _CANONICAL_NONNEGATIVE + if not isinstance(value, str) or pattern.fullmatch(value) is None: + qualifier = "positive " if positive else "non-negative " + raise ValidationError(f"{field_name} must be a canonical {qualifier}integer string") + parsed = int(value) + if parsed > _INT64_MAX: + raise ValidationError(f"{field_name} exceeds signed int64") + return parsed + + +@dataclass(frozen=True, order=True) +class EventSchemaRef: + schema_id: str + schema_version: str + + def __post_init__(self) -> None: + if not isinstance(self.schema_id, str) or _SCHEMA_ID.fullmatch(self.schema_id) is None: + raise ValidationError("schema_id must be a puresaber schema ID") + if ( + not isinstance(self.schema_version, str) + or _SEMVER.fullmatch(self.schema_version) is None + ): + raise ValidationError("schema_version must be semantic version text") + + def to_contract(self) -> dict[str, str]: + return {"schema_id": self.schema_id, "schema_version": self.schema_version} + + @classmethod + def from_contract(cls, payload: Mapping[str, Any]) -> EventSchemaRef: + _closed_payload(payload, {"schema_id", "schema_version"}, "event schema reference") + return cls(schema_id=payload["schema_id"], schema_version=payload["schema_version"]) + + +@dataclass(frozen=True, order=True) +class LineageRef: + role: str + snapshot_id: str + logical_sha256: str + + def __post_init__(self) -> None: + _required_text(self.role, "role") + _require_snapshot_id(self.snapshot_id, "snapshot_id") + _require_hash(self.logical_sha256, "logical_sha256") + + def to_contract(self) -> dict[str, str]: + return { + "role": self.role, + "snapshot_id": self.snapshot_id, + "logical_sha256": self.logical_sha256, + } + + +@dataclass(frozen=True) +class EventBarPartitionEvidence: + relative_path: str + source: str + instrument_id: str + session_id: str + first_sequence: int + last_sequence: int + first_event_id: str + last_event_id: str + event_count: int + source_selection_sha256: str + + def __post_init__(self) -> None: + for name in ( + "relative_path", + "source", + "instrument_id", + "session_id", + "first_event_id", + "last_event_id", + ): + _required_text(getattr(self, name), name) + for name in ("first_sequence", "last_sequence", "event_count"): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int): + raise ValidationError(f"{name} must be an integer") + if value < (1 if name == "event_count" else 0) or value > _INT64_MAX: + raise ValidationError(f"{name} is outside its signed-int64 contract") + if self.last_sequence < self.first_sequence: + raise ValidationError("event evidence sequence range is reversed") + _require_hash(self.source_selection_sha256, "source_selection_sha256") + + @property + def stream_key(self) -> tuple[str, str, str]: + return self.source, self.instrument_id, self.session_id + + def to_contract(self) -> dict[str, str]: + return { + "relative_path": self.relative_path, + "source": self.source, + "instrument_id": self.instrument_id, + "session_id": self.session_id, + "first_sequence": str(self.first_sequence), + "last_sequence": str(self.last_sequence), + "first_event_id": self.first_event_id, + "last_event_id": self.last_event_id, + "event_count": str(self.event_count), + "source_selection_sha256": self.source_selection_sha256, + } + + @classmethod + def from_contract(cls, payload: Mapping[str, Any]) -> EventBarPartitionEvidence: + fields = { + "relative_path", + "source", + "instrument_id", + "session_id", + "first_sequence", + "last_sequence", + "first_event_id", + "last_event_id", + "event_count", + "source_selection_sha256", + } + _closed_payload(payload, fields, "event-bar partition evidence") + return cls( + relative_path=payload["relative_path"], + source=payload["source"], + instrument_id=payload["instrument_id"], + session_id=payload["session_id"], + first_sequence=_canonical_integer( + payload["first_sequence"], "first_sequence", positive=False + ), + last_sequence=_canonical_integer( + payload["last_sequence"], "last_sequence", positive=False + ), + first_event_id=payload["first_event_id"], + last_event_id=payload["last_event_id"], + event_count=_canonical_integer(payload["event_count"], "event_count", positive=True), + source_selection_sha256=payload["source_selection_sha256"], + ) + + +@dataclass(frozen=True) +class CuratedAggregation: + calendar_id: str + session_policy_version: str + kind: Literal["fixed_time_bar", "session_bar", "event_bar"] + recipe_version: str + market_context_snapshot_id: str + market_context_logical_sha256: str + source_event_schemas: tuple[EventSchemaRef, ...] + interval_ns: int | None = None + session_rollup: Literal["session", "trading_day"] | None = None + event_bar_basis: Literal["trade_count", "base_volume", "quote_notional"] | None = None + event_bar_threshold: FixedPoint | None = None + partition_evidence: tuple[EventBarPartitionEvidence, ...] | None = None + + def __post_init__(self) -> None: + for name in ("calendar_id", "session_policy_version", "recipe_version"): + _required_text(getattr(self, name), name) + if self.kind not in {"fixed_time_bar", "session_bar", "event_bar"}: + raise ValidationError("unsupported Curated aggregation kind") + _require_snapshot_id(self.market_context_snapshot_id, "market_context_snapshot_id") + _require_hash(self.market_context_logical_sha256, "market_context_logical_sha256") + schemas = tuple(self.source_event_schemas) + if not schemas or schemas != tuple(sorted(set(schemas))): + raise ValidationError("source_event_schemas must be non-empty, unique, and sorted") + object.__setattr__(self, "source_event_schemas", schemas) + + if self.kind == "fixed_time_bar": + if ( + isinstance(self.interval_ns, bool) + or not isinstance(self.interval_ns, int) + or not 1 <= self.interval_ns <= _INT64_MAX + ): + raise ValidationError("fixed_time_bar requires positive int64 interval_ns") + if any( + value is not None + for value in ( + self.session_rollup, + self.event_bar_basis, + self.event_bar_threshold, + self.partition_evidence, + ) + ): + raise ValidationError("fixed_time_bar contains fields for another aggregation kind") + elif self.kind == "session_bar": + if self.session_rollup not in {"session", "trading_day"}: + raise ValidationError("session_bar requires session_rollup") + if any( + value is not None + for value in ( + self.interval_ns, + self.event_bar_basis, + self.event_bar_threshold, + self.partition_evidence, + ) + ): + raise ValidationError("session_bar contains fields for another aggregation kind") + else: + if self.event_bar_basis not in {"trade_count", "base_volume", "quote_notional"}: + raise ValidationError("event_bar requires an event_bar_basis") + if not isinstance(self.event_bar_threshold, FixedPoint) or not ( + self.event_bar_threshold.is_positive() + ): + raise ValidationError("event_bar requires a positive FixedPoint threshold") + if self.interval_ns is not None or self.session_rollup is not None: + raise ValidationError("event_bar contains fields for another aggregation kind") + evidence = tuple(self.partition_evidence or ()) + if not evidence: + raise ValidationError("event_bar requires partition evidence") + selection_hashes = [item.source_selection_sha256 for item in evidence] + if len(selection_hashes) != len(set(selection_hashes)): + raise ValidationError("event-bar selection hashes must be globally unique") + ordered = tuple( + sorted( + evidence, + key=lambda item: ( + item.stream_key, + item.first_sequence, + item.last_sequence, + item.relative_path, + ), + ) + ) + if evidence != ordered: + raise ValidationError("event-bar partition evidence must be canonically sorted") + previous_by_stream: dict[tuple[str, str, str], EventBarPartitionEvidence] = {} + for item in evidence: + previous = previous_by_stream.get(item.stream_key) + if previous is not None and item.first_sequence <= previous.last_sequence: + raise ValidationError("event-bar evidence ranges overlap within one stream") + previous_by_stream[item.stream_key] = item + object.__setattr__(self, "partition_evidence", evidence) + + def to_contract(self) -> dict[str, Any]: + threshold = ( + { + "units": str(self.event_bar_threshold.units), + "scale": self.event_bar_threshold.scale, + } + if self.event_bar_threshold is not None + else None + ) + return { + "calendar_id": self.calendar_id, + "session_policy_version": self.session_policy_version, + "kind": self.kind, + "recipe_version": self.recipe_version, + "interval_ns": str(self.interval_ns) if self.interval_ns is not None else None, + "session_rollup": self.session_rollup, + "event_bar_basis": self.event_bar_basis, + "event_bar_threshold": threshold, + "market_context_snapshot_id": self.market_context_snapshot_id, + "market_context_logical_sha256": self.market_context_logical_sha256, + "source_event_schemas": [item.to_contract() for item in self.source_event_schemas], + "partition_evidence": ( + [item.to_contract() for item in self.partition_evidence] + if self.partition_evidence is not None + else None + ), + } + + @classmethod + def from_contract(cls, payload: Mapping[str, Any]) -> CuratedAggregation: + fields = { + "calendar_id", + "session_policy_version", + "kind", + "recipe_version", + "interval_ns", + "session_rollup", + "event_bar_basis", + "event_bar_threshold", + "market_context_snapshot_id", + "market_context_logical_sha256", + "source_event_schemas", + "partition_evidence", + } + _closed_payload(payload, fields, "Curated aggregation") + interval = payload["interval_ns"] + threshold = payload["event_bar_threshold"] + if threshold is not None: + _closed_payload(threshold, {"units", "scale"}, "event-bar threshold") + units = _canonical_integer(threshold["units"], "threshold units", positive=True) + scale = threshold["scale"] + if isinstance(scale, bool) or not isinstance(scale, int): + raise ValidationError("threshold scale must be an integer") + threshold_value: FixedPoint | None = FixedPoint(units=units, scale=scale) + else: + threshold_value = None + raw_schemas = payload["source_event_schemas"] + if not isinstance(raw_schemas, list): + raise ValidationError("source_event_schemas must be an array") + raw_evidence = payload["partition_evidence"] + if raw_evidence is not None and not isinstance(raw_evidence, list): + raise ValidationError("partition_evidence must be an array or null") + return cls( + calendar_id=payload["calendar_id"], + session_policy_version=payload["session_policy_version"], + kind=payload["kind"], + recipe_version=payload["recipe_version"], + interval_ns=( + _canonical_integer(interval, "interval_ns", positive=True) + if interval is not None + else None + ), + session_rollup=payload["session_rollup"], + event_bar_basis=payload["event_bar_basis"], + event_bar_threshold=threshold_value, + market_context_snapshot_id=payload["market_context_snapshot_id"], + market_context_logical_sha256=payload["market_context_logical_sha256"], + source_event_schemas=tuple(EventSchemaRef.from_contract(item) for item in raw_schemas), + partition_evidence=( + tuple(EventBarPartitionEvidence.from_contract(item) for item in raw_evidence) + if raw_evidence is not None + else None + ), + ) + + +@dataclass(frozen=True, eq=False) +class VerifiedFactorInput: + layer: Literal["curated", "normalized"] + source_snapshot_id: str + source_logical_sha256: str + selection_logical_sha256: str + event_schemas: tuple[EventSchemaRef, ...] + table: pa.Table = field(repr=False) + calendar_id: str = "" + session_policy_version: str = "" + market_context_snapshot_id: str = "" + market_context_logical_sha256: str = "" + lineage: tuple[LineageRef, ...] = () + aggregation: CuratedAggregation | None = None + schema_id: str = VERIFIED_FACTOR_INPUT_SCHEMA_ID + + def __post_init__(self) -> None: + if self.schema_id != VERIFIED_FACTOR_INPUT_SCHEMA_ID: + raise ValidationError("unsupported VerifiedFactorInput schema") + if self.layer not in {"curated", "normalized"}: + raise ValidationError("unsupported verified input layer") + _require_snapshot_id(self.source_snapshot_id, "source_snapshot_id") + _require_hash(self.source_logical_sha256, "source_logical_sha256") + _require_hash(self.selection_logical_sha256, "selection_logical_sha256") + _required_text(self.calendar_id, "calendar_id") + _required_text(self.session_policy_version, "session_policy_version") + _require_snapshot_id(self.market_context_snapshot_id, "market_context_snapshot_id") + _require_hash(self.market_context_logical_sha256, "market_context_logical_sha256") + schemas = tuple(self.event_schemas) + if not schemas or schemas != tuple(sorted(set(schemas))): + raise ValidationError("event_schemas must be non-empty, unique, and sorted") + if not isinstance(self.table, pa.Table) or self.table.num_rows <= 0: + raise ValidationError("verified input table must be a non-empty Arrow table") + lineage = tuple(self.lineage) + if not lineage or lineage != tuple(sorted(lineage)): + raise ValidationError("lineage must be non-empty and canonically ordered") + if self.layer == "curated": + if self.aggregation is None: + raise ValidationError("Curated verified input requires aggregation metadata") + elif self.aggregation is not None: + raise ValidationError("Normalized verified input cannot contain aggregation metadata") + object.__setattr__(self, "event_schemas", schemas) + object.__setattr__(self, "lineage", lineage) + + @property + def arrow_schema_sha256(self) -> str: + import hashlib + + return hashlib.sha256(self.table.schema.serialize().to_pybytes()).hexdigest() + + def to_contract(self) -> dict[str, Any]: + return { + "schema_id": self.schema_id, + "layer": self.layer, + "source_snapshot_id": self.source_snapshot_id, + "source_logical_sha256": self.source_logical_sha256, + "selection_logical_sha256": self.selection_logical_sha256, + "event_schemas": [item.to_contract() for item in self.event_schemas], + "calendar_id": self.calendar_id, + "session_policy_version": self.session_policy_version, + "market_context_snapshot_id": self.market_context_snapshot_id, + "market_context_logical_sha256": self.market_context_logical_sha256, + "lineage": [item.to_contract() for item in self.lineage], + "rows": str(self.table.num_rows), + "arrow_schema_sha256": self.arrow_schema_sha256, + "aggregation": self.aggregation.to_contract() if self.aggregation else None, + } diff --git a/src/quant_data_kit/research_inputs_v2.py b/src/quant_data_kit/research_inputs_v2.py new file mode 100644 index 0000000..819ab98 --- /dev/null +++ b/src/quant_data_kit/research_inputs_v2.py @@ -0,0 +1,963 @@ +"""Fail-closed M8 factories for immutable Curated and Normalized research inputs.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq +from pyarrow import ipc + +from quant_data_kit.curated import build_event_bars, load_curated_snapshot +from quant_data_kit.data_lake import ( + StoragePolicy, + _lake_lock, + _mkdir_in_lake, + _publish_tree_entry, + _resolved_lake_root, + _stable_staging_directory, + _validate_lake_path, + load_normalized_snapshot, + read_normalized_events, + require_collection_capacity, +) +from quant_data_kit.domain_v2 import ( + AssetClass, + InstrumentSpec, + MarginMode, + SessionPhase, + TradingSession, + dataclass_payload, +) +from quant_data_kit.exceptions import ValidationError +from quant_data_kit.fixed_point import FixedPoint +from quant_data_kit.l2_replay import L2ReplayError, replay_l2 +from quant_data_kit.market_clock_v2 import MarketClock +from quant_data_kit.research_contracts_v2 import ( + MARKET_CONTEXT_SCHEMA_ID, + CuratedAggregation, + EventBarPartitionEvidence, + EventSchemaRef, + LineageRef, + VerifiedFactorInput, +) +from quant_data_kit.schemas_v2 import ( + BAR_EVENT_SCHEMA_ID, + BOOK_DELTA_EVENT_SCHEMA_ID, + BOOK_SNAPSHOT_EVENT_SCHEMA_ID, + CORPORATE_ACTION_EVENT_SCHEMA_ID, + FUNDING_RATE_EVENT_SCHEMA_ID, + MARK_PRICE_EVENT_SCHEMA_ID, + QUOTE_EVENT_SCHEMA_ID, + SCHEMA_VERSION_V2, + STATUS_EVENT_SCHEMA_ID, + TRADE_EVENT_SCHEMA_ID, + get_arrow_schema, + validate_arrow_table, + validate_json_record, +) + +_SNAPSHOT_ID = re.compile(r"^sha256-[0-9a-f]{64}$") +_EVENT_TYPE_BY_SCHEMA = { + BOOK_DELTA_EVENT_SCHEMA_ID: "book_delta", + BOOK_SNAPSHOT_EVENT_SCHEMA_ID: "book_snapshot", + CORPORATE_ACTION_EVENT_SCHEMA_ID: "corporate_action", + FUNDING_RATE_EVENT_SCHEMA_ID: "funding_rate", + MARK_PRICE_EVENT_SCHEMA_ID: "mark_price", + QUOTE_EVENT_SCHEMA_ID: "quote", + STATUS_EVENT_SCHEMA_ID: "status", + TRADE_EVENT_SCHEMA_ID: "trade", +} +_INT64_MAX = 2**63 - 1 + + +def _canonical(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _hash_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _utc(value: str | datetime, field_name: str) -> datetime: + try: + parsed = ( + value + if isinstance(value, datetime) + else datetime.fromisoformat(value.replace("Z", "+00:00")) + ) + except (TypeError, ValueError) as exc: + raise ValidationError(f"{field_name} must be a valid UTC timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() != timezone.utc.utcoffset(parsed): + raise ValidationError(f"{field_name} must be UTC-aware") + return parsed.astimezone(timezone.utc) + + +def _parse_optional_utc(value: Any, field_name: str) -> datetime | None: + return None if value is None else _utc(value, field_name) + + +def _fixed(payload: Mapping[str, Any], field_name: str) -> FixedPoint: + if not isinstance(payload, Mapping) or set(payload) != {"units", "scale"}: + raise ValidationError(f"{field_name} must be a closed FixedPoint object") + return FixedPoint(units=payload["units"], scale=payload["scale"]) + + +def _arrow_ready(record: Mapping[str, Any], schema: pa.Schema) -> dict[str, Any]: + result = dict(record) + for field_definition in schema: + value = result.get(field_definition.name) + if value is None: + continue + if pa.types.is_timestamp(field_definition.type): + result[field_definition.name] = _utc(value, field_definition.name) + elif pa.types.is_date32(field_definition.type): + result[field_definition.name] = ( + value if isinstance(value, date) else date.fromisoformat(str(value)) + ) + return result + + +def _table_logical_sha256(table: pa.Table) -> str: + combined = table.combine_chunks() + sink = pa.BufferOutputStream() + with ipc.new_stream(sink, combined.schema) as writer: + writer.write_table(combined) + return _hash_bytes(sink.getvalue().to_pybytes()) + + +def _record_selection_sha256(schema_id: str, rows: Sequence[Mapping[str, Any]]) -> str: + payload = { + "algorithm": "puresaber.event-selection-canonical-json@1.0.0", + "schema_id": schema_id, + "schema_version": SCHEMA_VERSION_V2, + "records": [dict(row) for row in rows], + } + return _hash_bytes(_canonical(payload)) + + +@dataclass(frozen=True) +class MarketContextSnapshot: + snapshot_id: str + logical_sha256: str + calendar_id: str + session_policy_version: str + instruments: tuple[InstrumentSpec, ...] + sessions: tuple[TradingSession, ...] + schema_id: str = MARKET_CONTEXT_SCHEMA_ID + + def identity(self) -> dict[str, Any]: + return { + "schema_id": self.schema_id, + "calendar_id": self.calendar_id, + "session_policy_version": self.session_policy_version, + "instruments": [dataclass_payload(item) for item in self.instruments], + "sessions": [dataclass_payload(item) for item in self.sessions], + } + + def manifest(self) -> dict[str, Any]: + return { + **self.identity(), + "snapshot_id": self.snapshot_id, + "logical_sha256": self.logical_sha256, + } + + +def _market_context_identity( + *, + calendar_id: str, + session_policy_version: str, + instruments: tuple[InstrumentSpec, ...], + sessions: tuple[TradingSession, ...], +) -> dict[str, Any]: + return { + "schema_id": MARKET_CONTEXT_SCHEMA_ID, + "calendar_id": calendar_id, + "session_policy_version": session_policy_version, + "instruments": [dataclass_payload(item) for item in instruments], + "sessions": [dataclass_payload(item) for item in sessions], + } + + +def _validate_market_context_values( + calendar_id: str, + session_policy_version: str, + instruments: tuple[InstrumentSpec, ...], + sessions: tuple[TradingSession, ...], +) -> None: + if not isinstance(calendar_id, str) or not calendar_id.strip(): + raise ValidationError("calendar_id is required") + if not isinstance(session_policy_version, str) or not session_policy_version.strip(): + raise ValidationError("session_policy_version is required") + if not instruments or not sessions: + raise ValidationError("market context requires instruments and sessions") + if any(item.calendar_id != calendar_id for item in instruments): + raise ValidationError("all instruments must use the market-context calendar") + if any(item.calendar_id != calendar_id for item in sessions): + raise ValidationError("all sessions must use the market-context calendar") + instrument_keys = [ + (item.instrument_id, item.effective_from, item.available_at) for item in instruments + ] + if len(instrument_keys) != len(set(instrument_keys)): + raise ValidationError("market context contains duplicate instrument versions") + session_ids = [item.session_id for item in sessions] + if len(session_ids) != len(set(session_ids)): + raise ValidationError("market context contains duplicate session IDs") + MarketClock(calendar_id, sessions) + + +def create_market_context_snapshot( + root: Path, + *, + calendar_id: str, + session_policy_version: str, + instruments: Iterable[InstrumentSpec], + sessions: Iterable[TradingSession], + policy: StoragePolicy | None = None, +) -> MarketContextSnapshot: + """Persist a deterministic, content-addressed instrument/session context.""" + materialized_instruments = tuple(instruments) + materialized_sessions = tuple(sessions) + if not all(isinstance(item, InstrumentSpec) for item in materialized_instruments): + raise ValidationError("instruments must contain InstrumentSpec values") + if not all(isinstance(item, TradingSession) for item in materialized_sessions): + raise ValidationError("sessions must contain TradingSession values") + ordered_instruments = tuple( + sorted( + materialized_instruments, + key=lambda item: (item.instrument_id, item.effective_from, item.available_at), + ) + ) + ordered_sessions = tuple( + sorted(materialized_sessions, key=lambda item: (item.opens_at, item.session_id)) + ) + _validate_market_context_values( + calendar_id, + session_policy_version, + ordered_instruments, + ordered_sessions, + ) + identity = _market_context_identity( + calendar_id=calendar_id, + session_policy_version=session_policy_version, + instruments=ordered_instruments, + sessions=ordered_sessions, + ) + logical_sha256 = _hash_bytes(_canonical(identity)) + snapshot_id = f"sha256-{logical_sha256}" + snapshot = MarketContextSnapshot( + snapshot_id=snapshot_id, + logical_sha256=logical_sha256, + calendar_id=calendar_id, + session_policy_version=session_policy_version, + instruments=ordered_instruments, + sessions=ordered_sessions, + ) + lake_root = _resolved_lake_root(root, create=True) + context_root = _mkdir_in_lake(lake_root, lake_root / "market-context") + snapshots_root = _mkdir_in_lake(lake_root, context_root / "snapshots") + target = snapshots_root / snapshot_id + resolved_policy = policy or StoragePolicy() + require_collection_capacity( + lake_root, + projected_write_bytes=len(_canonical(snapshot.manifest())), + policy=resolved_policy, + ) + with _stable_staging_directory( + lake_root, + context_root / "staging", + namespace="market-context", + identity=identity, + ) as stage: + (stage / "manifest.json").write_text( + json.dumps(snapshot.manifest(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + with _lake_lock(lake_root, "market-context", {"snapshot_id": snapshot_id}): + if target.exists(): + existing = load_market_context_snapshot(lake_root, snapshot_id) + if existing.manifest() != snapshot.manifest(): + raise ValidationError("market-context snapshot collision") + return existing + _publish_tree_entry(lake_root, stage, target, policy=resolved_policy) + stage = target + return load_market_context_snapshot(lake_root, snapshot_id) + + +def _instrument_from_payload(payload: Mapping[str, Any]) -> InstrumentSpec: + required = { + "instrument_id", + "asset_class", + "product_type", + "venue", + "native_symbol", + "base_currency", + "quote_currency", + "settlement_currency", + "price_tick", + "quantity_step", + "contract_multiplier", + "calendar_id", + "margin_mode", + "inverse", + "effective_from", + "effective_to", + "available_at", + "superseded_at", + "underlying_id", + "expiry_date", + "metadata", + } + if not isinstance(payload, Mapping) or set(payload) != required: + raise ValidationError("instrument context record is not closed") + metadata = payload["metadata"] + if not isinstance(metadata, Mapping): + raise ValidationError("instrument metadata must be an object") + return InstrumentSpec( + instrument_id=payload["instrument_id"], + asset_class=AssetClass(payload["asset_class"]), + product_type=payload["product_type"], + venue=payload["venue"], + native_symbol=payload["native_symbol"], + base_currency=payload["base_currency"], + quote_currency=payload["quote_currency"], + settlement_currency=payload["settlement_currency"], + price_tick=_fixed(payload["price_tick"], "price_tick"), + quantity_step=_fixed(payload["quantity_step"], "quantity_step"), + contract_multiplier=_fixed(payload["contract_multiplier"], "contract_multiplier"), + calendar_id=payload["calendar_id"], + margin_mode=MarginMode(payload["margin_mode"]), + inverse=payload["inverse"], + effective_from=_utc(payload["effective_from"], "effective_from"), + effective_to=_parse_optional_utc(payload["effective_to"], "effective_to"), + available_at=_utc(payload["available_at"], "available_at"), + superseded_at=_parse_optional_utc(payload["superseded_at"], "superseded_at"), + underlying_id=payload["underlying_id"], + expiry_date=( + date.fromisoformat(payload["expiry_date"]) + if payload["expiry_date"] is not None + else None + ), + metadata=dict(metadata), + ) + + +def _session_from_payload(payload: Mapping[str, Any]) -> TradingSession: + required = { + "session_id", + "calendar_id", + "venue", + "trading_day", + "phase", + "opens_at", + "closes_at", + "available_at", + "superseded_at", + } + if not isinstance(payload, Mapping) or set(payload) != required: + raise ValidationError("session context record is not closed") + return TradingSession( + session_id=payload["session_id"], + calendar_id=payload["calendar_id"], + venue=payload["venue"], + trading_day=date.fromisoformat(payload["trading_day"]), + phase=SessionPhase(payload["phase"]), + opens_at=_utc(payload["opens_at"], "opens_at"), + closes_at=_utc(payload["closes_at"], "closes_at"), + available_at=_utc(payload["available_at"], "available_at"), + superseded_at=_parse_optional_utc(payload["superseded_at"], "superseded_at"), + ) + + +def load_market_context_snapshot(root: Path, snapshot_id: str) -> MarketContextSnapshot: + lake_root = _resolved_lake_root(root, create=False) + if not isinstance(snapshot_id, str) or _SNAPSHOT_ID.fullmatch(snapshot_id) is None: + raise ValidationError("market-context reads require a content-addressed snapshot ID") + snapshot_dir = _validate_lake_path( + lake_root, + lake_root / "market-context" / "snapshots" / snapshot_id, + allow_missing=False, + ) + manifest_path = _validate_lake_path( + lake_root, snapshot_dir / "manifest.json", allow_missing=False + ) + if not manifest_path.is_file(): + raise ValidationError("market-context manifest is missing") + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValidationError("market-context manifest is malformed") from exc + fields = { + "schema_id", + "snapshot_id", + "logical_sha256", + "calendar_id", + "session_policy_version", + "instruments", + "sessions", + } + if not isinstance(payload, Mapping) or set(payload) != fields: + raise ValidationError("market-context manifest is not closed") + if payload["schema_id"] != MARKET_CONTEXT_SCHEMA_ID or payload["snapshot_id"] != snapshot_id: + raise ValidationError("market-context identity mismatch") + try: + instruments = tuple(_instrument_from_payload(item) for item in payload["instruments"]) + sessions = tuple(_session_from_payload(item) for item in payload["sessions"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("market-context records are malformed") from exc + _validate_market_context_values( + payload["calendar_id"], payload["session_policy_version"], instruments, sessions + ) + if instruments != tuple( + sorted( + instruments, + key=lambda item: (item.instrument_id, item.effective_from, item.available_at), + ) + ) or sessions != tuple(sorted(sessions, key=lambda item: (item.opens_at, item.session_id))): + raise ValidationError("market-context records are not canonically sorted") + identity = _market_context_identity( + calendar_id=payload["calendar_id"], + session_policy_version=payload["session_policy_version"], + instruments=instruments, + sessions=sessions, + ) + logical_sha256 = _hash_bytes(_canonical(identity)) + if payload["logical_sha256"] != logical_sha256 or snapshot_id != f"sha256-{logical_sha256}": + raise ValidationError("market-context logical hash changed") + actual_files = { + path.relative_to(snapshot_dir) for path in snapshot_dir.rglob("*") if path.is_file() + } + if actual_files != {Path("manifest.json")}: + raise ValidationError("market-context snapshot contains unexpected files") + return MarketContextSnapshot( + snapshot_id=snapshot_id, + logical_sha256=logical_sha256, + calendar_id=payload["calendar_id"], + session_policy_version=payload["session_policy_version"], + instruments=instruments, + sessions=sessions, + ) + + +def _active_instrument( + context: MarketContextSnapshot, + instrument_id: str, + event_time: datetime, + available_at: datetime, +) -> InstrumentSpec: + candidates = [ + item + for item in context.instruments + if item.instrument_id == instrument_id + and item.effective_from <= event_time + and (item.effective_to is None or event_time < item.effective_to) + and item.available_at <= available_at + and (item.superseded_at is None or available_at < item.superseded_at) + ] + if not candidates: + raise ValidationError(f"no PIT-valid InstrumentSpec for {instrument_id}") + newest_effective = max(item.effective_from for item in candidates) + selected = [item for item in candidates if item.effective_from == newest_effective] + if len(selected) != 1: + raise ValidationError(f"ambiguous PIT InstrumentSpec for {instrument_id}") + return selected[0] + + +def _context_session( + context: MarketContextSnapshot, + session_id: str, + event_time: datetime, + available_at: datetime, + trading_day: date, + *, + allow_close: bool, +) -> TradingSession: + matches = [item for item in context.sessions if item.session_id == session_id] + if len(matches) != 1: + raise ValidationError(f"market context must contain exactly one session {session_id}") + session = matches[0] + upper_ok = event_time <= session.closes_at if allow_close else event_time < session.closes_at + if not session.opens_at <= event_time or not upper_ok: + raise ValidationError(f"event time is outside session {session_id}") + if session.trading_day != trading_day: + raise ValidationError(f"trading_day does not match session {session_id}") + if session.available_at > available_at or ( + session.superseded_at is not None and available_at >= session.superseded_at + ): + raise ValidationError(f"session {session_id} is not PIT-valid") + return session + + +def _validate_context_record( + context: MarketContextSnapshot, + record: Mapping[str, Any], + *, + bar: bool, +) -> tuple[InstrumentSpec, TradingSession]: + event_time = _utc(record["event_time"], "event_time") + received_at = _utc(record["received_at"], "received_at") + available_at = _utc(record["available_at"], "available_at") + if event_time > received_at or received_at > available_at: + raise ValidationError("event PIT timestamps are not monotonic") + trading_day_value = record["trading_day"] + trading_day = ( + trading_day_value + if isinstance(trading_day_value, date) + else date.fromisoformat(str(trading_day_value)) + ) + instrument = _active_instrument(context, str(record["instrument_id"]), event_time, available_at) + session = _context_session( + context, + str(record["session_id"]), + event_time, + available_at, + trading_day, + allow_close=bar, + ) + if instrument.calendar_id != context.calendar_id or session.calendar_id != context.calendar_id: + raise ValidationError("record context calendar mismatch") + if instrument.venue != session.venue: + raise ValidationError("instrument venue and session venue differ") + return instrument, session + + +def _ordered_records( + records: Sequence[tuple[EventSchemaRef, dict[str, Any]]], +) -> list[tuple[EventSchemaRef, dict[str, Any]]]: + return sorted( + records, + key=lambda item: ( + str(item[1]["source"]), + str(item[1]["instrument_id"]), + str(item[1]["session_id"]), + _utc(item[1]["event_time"], "event_time"), + int(item[1]["sequence"]), + str(item[1]["event_id"]), + item[0].schema_id, + ), + ) + + +def _validate_event_order(records: Sequence[tuple[EventSchemaRef, dict[str, Any]]]) -> None: + event_ids: set[str] = set() + previous_by_stream: dict[tuple[str, str, str, str], tuple[datetime, int, str]] = {} + l2_streams: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + for schema_ref, record in records: + event_id = str(record["event_id"]) + if event_id in event_ids: + raise ValidationError(f"duplicate event_id in verified selection: {event_id}") + event_ids.add(event_id) + event_type = str(record["event_type"]) + domain = "book" if event_type in {"book_snapshot", "book_delta"} else schema_ref.schema_id + stream = ( + str(record["source"]), + str(record["instrument_id"]), + str(record["session_id"]), + domain, + ) + identity = ( + _utc(record["event_time"], "event_time"), + int(record["sequence"]), + event_id, + ) + previous = previous_by_stream.get(stream) + if previous is not None and identity <= previous: + raise ValidationError(f"event stream is not strictly ordered: {stream}") + if previous is not None and int(record["sequence"]) <= previous[1]: + raise ValidationError(f"event sequence does not advance: {stream}") + previous_by_stream[stream] = identity + if domain == "book": + l2_streams[stream[:3]].append(record) + for stream, l2_records in l2_streams.items(): + try: + replay_l2(l2_records, capture_all_checkpoints=False) + except (L2ReplayError, ValidationError) as exc: + raise ValidationError(f"L2 selection failed replay for {stream}: {exc}") from exc + + +def _event_union_table(records: Sequence[tuple[EventSchemaRef, dict[str, Any]]]) -> pa.Table: + schema_count = len({item.schema_id for item, _ in records}) + field_types: dict[str, pa.DataType] = {} + field_nullability: dict[str, bool] = {} + field_occurrences: dict[str, set[str]] = defaultdict(set) + field_order: list[str] = [] + for schema_ref, _ in records: + schema = get_arrow_schema(schema_ref.schema_id, schema_ref.schema_version) + for field_definition in schema: + existing = field_types.get(field_definition.name) + if existing is not None and existing != field_definition.type: + raise ValidationError(f"event schemas disagree on field {field_definition.name}") + if field_definition.name not in field_types: + field_order.append(field_definition.name) + field_types[field_definition.name] = field_definition.type + field_nullability[field_definition.name] = field_definition.nullable + else: + field_nullability[field_definition.name] |= field_definition.nullable + field_occurrences[field_definition.name].add(schema_ref.schema_id) + fields = [pa.field("event_schema_id", pa.string(), nullable=False)] + for name in field_order: + fields.append( + pa.field( + name, + field_types[name], + nullable=field_nullability[name] or len(field_occurrences[name]) < schema_count, + ) + ) + union_schema = pa.schema(fields) + ready: list[dict[str, Any]] = [] + for schema_ref, record in records: + source_schema = get_arrow_schema(schema_ref.schema_id, schema_ref.schema_version) + converted = _arrow_ready(record, source_schema) + row = {name: converted.get(name) for name in field_order} + row["event_schema_id"] = schema_ref.schema_id + ready.append(row) + return pa.Table.from_pylist(ready, schema=union_schema).combine_chunks() + + +def _normalize_schema_refs( + event_schemas: Iterable[EventSchemaRef | Mapping[str, Any]], +) -> tuple[EventSchemaRef, ...]: + refs = tuple( + item if isinstance(item, EventSchemaRef) else EventSchemaRef.from_contract(item) + for item in event_schemas + ) + ordered = tuple(sorted(set(refs))) + if not refs or refs != ordered: + raise ValidationError("event_schemas must be non-empty, unique, and sorted") + if any(item.schema_version != SCHEMA_VERSION_V2 for item in refs): + raise ValidationError("only event schema version 2.0.0 is certified") + if any( + item.schema_id == BAR_EVENT_SCHEMA_ID or item.schema_id not in _EVENT_TYPE_BY_SCHEMA + for item in refs + ): + raise ValidationError("Normalized verified input accepts only non-Bar market events") + return refs + + +def load_verified_normalized_events( + root: Path, + snapshot_id: str, + event_schemas: Iterable[EventSchemaRef | Mapping[str, Any]], + market_context_snapshot_id: str, +) -> VerifiedFactorInput: + """Load a frozen market-event selection after complete snapshot and context verification.""" + refs = _normalize_schema_refs(event_schemas) + lake_root = _resolved_lake_root(root, create=False) + before = load_normalized_snapshot(lake_root, snapshot_id) + context_before = load_market_context_snapshot(lake_root, market_context_snapshot_id) + rows = read_normalized_events(lake_root, before.snapshot_id) + ref_by_type = {_EVENT_TYPE_BY_SCHEMA[item.schema_id]: item for item in refs} + selected: list[tuple[EventSchemaRef, dict[str, Any]]] = [] + counts = {item: 0 for item in refs} + for raw in rows: + event_type = str(raw.get("event_type")) + schema_ref = ref_by_type.get(event_type) + if schema_ref is None: + continue + record = dict(raw) + validate_json_record(schema_ref.schema_id, record, schema_ref.schema_version) + _validate_context_record(context_before, record, bar=False) + selected.append((schema_ref, record)) + counts[schema_ref] += 1 + missing = [item.schema_id for item, count in counts.items() if count == 0] + if missing: + raise ValidationError(f"Normalized snapshot lacks requested event schemas: {missing}") + ordered = _ordered_records(selected) + _validate_event_order(ordered) + table = _event_union_table(ordered) + after = load_normalized_snapshot(lake_root, snapshot_id) + context_after = load_market_context_snapshot(lake_root, market_context_snapshot_id) + if ( + before.snapshot_id, + before.logical_sha256, + before.partitions, + ) != (after.snapshot_id, after.logical_sha256, after.partitions): + raise ValidationError("Normalized snapshot changed while building verified input") + if context_before.manifest() != context_after.manifest(): + raise ValidationError("market context changed while building verified input") + return VerifiedFactorInput( + layer="normalized", + source_snapshot_id=before.snapshot_id, + source_logical_sha256=before.logical_sha256, + selection_logical_sha256=_table_logical_sha256(table), + event_schemas=refs, + table=table, + calendar_id=context_before.calendar_id, + session_policy_version=context_before.session_policy_version, + market_context_snapshot_id=context_before.snapshot_id, + market_context_logical_sha256=context_before.logical_sha256, + lineage=tuple( + sorted( + ( + LineageRef("market", before.snapshot_id, before.logical_sha256), + LineageRef( + "market_context", context_before.snapshot_id, context_before.logical_sha256 + ), + ) + ) + ), + ) + + +def _validate_bar_rows( + table: pa.Table, + aggregation: CuratedAggregation, + context: MarketContextSnapshot, +) -> list[dict[str, Any]]: + rows = table.to_pylist() + identities: set[tuple[str, datetime, int, str]] = set() + previous_by_instrument: dict[str, tuple[datetime, int, str]] = {} + sessions_by_day: dict[tuple[str, date], list[TradingSession]] = defaultdict(list) + for session in context.sessions: + sessions_by_day[(session.venue, session.trading_day)].append(session) + for row in rows: + ready = dict(row) + validate_json_record( + BAR_EVENT_SCHEMA_ID, + { + key: ( + value.isoformat().replace("+00:00", "Z") + if isinstance(value, datetime) + else value.isoformat() + if isinstance(value, date) + else value + ) + for key, value in ready.items() + }, + ) + if not bool(row["is_complete"]): + raise ValidationError("certified factor input rejects incomplete Bars") + bar_start = _utc(row["bar_start"], "bar_start") + bar_end = _utc(row["bar_end"], "bar_end") + event_time = _utc(row["event_time"], "event_time") + instrument, session = _validate_context_record(context, row, bar=True) + identity = ( + str(row["instrument_id"]), + event_time, + int(row["sequence"]), + str(row["event_id"]), + ) + if identity in identities: + raise ValidationError("Curated input contains duplicate Bar identity") + identities.add(identity) + prior = previous_by_instrument.get(str(row["instrument_id"])) + ordering = identity[1:] + if prior is not None and ordering <= prior: + raise ValidationError("Curated Bars are not strictly ordered per instrument") + previous_by_instrument[str(row["instrument_id"])] = ordering + if aggregation.kind == "fixed_time_bar": + duration = bar_end - bar_start + duration_ns = ( + duration.days * 86_400 + duration.seconds + ) * 1_000_000_000 + duration.microseconds * 1_000 + if duration_ns != aggregation.interval_ns: + raise ValidationError("Bar interval differs from Curated aggregation metadata") + if bar_start < session.opens_at or bar_end > session.closes_at: + raise ValidationError("fixed-time Bar crosses its session boundary") + elif aggregation.kind == "session_bar": + if aggregation.session_rollup == "session": + if bar_start != session.opens_at or bar_end != session.closes_at: + raise ValidationError("session Bar does not match its authoritative session") + else: + day_sessions = sessions_by_day[(instrument.venue, session.trading_day)] + expected_start = min(item.opens_at for item in day_sessions) + expected_end = max(item.closes_at for item in day_sessions) + if bar_start != expected_start or bar_end != expected_end: + raise ValidationError("trading-day Bar boundaries are not authoritative") + return rows + + +def _source_rows_for_evidence( + all_rows: Sequence[Mapping[str, Any]], evidence: EventBarPartitionEvidence +) -> list[dict[str, Any]]: + selected = [ + dict(row) + for row in all_rows + if row.get("event_type") == "trade" + and str(row["source"]) == evidence.source + and str(row["instrument_id"]) == evidence.instrument_id + and str(row["session_id"]) == evidence.session_id + and evidence.first_sequence <= int(row["sequence"]) <= evidence.last_sequence + ] + selected.sort( + key=lambda row: ( + _utc(row["event_time"], "event_time"), + int(row["sequence"]), + str(row["event_id"]), + ) + ) + if len(selected) != evidence.event_count: + raise ValidationError("event-bar evidence event_count does not match Normalized lineage") + if ( + int(selected[0]["sequence"]) != evidence.first_sequence + or int(selected[-1]["sequence"]) != evidence.last_sequence + or str(selected[0]["event_id"]) != evidence.first_event_id + or str(selected[-1]["event_id"]) != evidence.last_event_id + ): + raise ValidationError("event-bar evidence boundaries do not match Normalized lineage") + if ( + _record_selection_sha256(TRADE_EVENT_SCHEMA_ID, selected) + != evidence.source_selection_sha256 + ): + raise ValidationError("event-bar source selection hash changed") + return selected + + +def _verify_event_bars( + root: Path, + aggregation: CuratedAggregation, + normalized_snapshot_id: str, + table: pa.Table, + context: MarketContextSnapshot, +) -> None: + if aggregation.partition_evidence is None or aggregation.event_bar_threshold is None: + raise ValidationError("event-bar metadata is incomplete") + if aggregation.source_event_schemas != ( + EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + ): + raise ValidationError("M8 event bars currently require the frozen Trade schema") + source_rows = read_normalized_events(root, normalized_snapshot_id, event_type="trade") + output_sources = set(table.column("source").to_pylist()) + if len(output_sources) != 1: + raise ValidationError("event Bars must use one deterministic Curated source") + output_source = str(next(iter(output_sources))) + session_starts = {item.session_id: item.opens_at for item in context.sessions} + rebuilt: list[dict[str, Any]] = [] + for evidence in aggregation.partition_evidence: + selected = _source_rows_for_evidence(source_rows, evidence) + rebuilt.extend( + build_event_bars( + selected, + basis=aggregation.event_bar_basis, + threshold=aggregation.event_bar_threshold, + session_starts=session_starts, + source=output_source, + recipe_version=aggregation.recipe_version, + require_complete=True, + ) + ) + expected_table = pa.Table.from_pylist( + [_arrow_ready(item, get_arrow_schema(BAR_EVENT_SCHEMA_ID)) for item in rebuilt], + schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID), + ) + sort_keys = [ + ("instrument_id", "ascending"), + ("event_time", "ascending"), + ("sequence", "ascending"), + ("event_id", "ascending"), + ] + expected_table = expected_table.take(pc.sort_indices(expected_table, sort_keys=sort_keys)) + actual_table = table.take(pc.sort_indices(table, sort_keys=sort_keys)) + if not expected_table.equals(actual_table, check_metadata=True): + raise ValidationError("event Bars do not recompute from their Normalized evidence") + + +def load_verified_curated_bars( + root: Path, + dataset: str, + snapshot_id: str, +) -> VerifiedFactorInput: + """Load only M8-certified Curated Bars; legacy Curated snapshots fail closed.""" + lake_root = _resolved_lake_root(root, create=False) + before = load_curated_snapshot(lake_root, dataset, snapshot_id) + aggregation = before.aggregation + if not isinstance(aggregation, CuratedAggregation): + raise ValidationError("legacy-curated-not-m8-certified") + if aggregation.recipe_version != before.recipe_version: + raise ValidationError("Curated recipe and aggregation recipe differ") + context_before = load_market_context_snapshot(lake_root, aggregation.market_context_snapshot_id) + if ( + aggregation.market_context_logical_sha256 != context_before.logical_sha256 + or aggregation.calendar_id != context_before.calendar_id + or aggregation.session_policy_version != context_before.session_policy_version + ): + raise ValidationError("Curated aggregation market-context binding changed") + snapshot_dir = _validate_lake_path( + lake_root, + lake_root / "curated" / dataset / "snapshots" / snapshot_id, + allow_missing=False, + ) + tables: list[pa.Table] = [] + for partition in before.partitions: + path = _validate_lake_path( + lake_root, snapshot_dir / partition.relative_path, allow_missing=False + ) + table = pq.ParquetFile(path).read() + validate_arrow_table(BAR_EVENT_SCHEMA_ID, table) + tables.append(table) + if not tables: + raise ValidationError("Curated snapshot contains no Bar partitions") + combined = pa.concat_tables(tables).combine_chunks() + sort_keys = [ + ("instrument_id", "ascending"), + ("event_time", "ascending"), + ("sequence", "ascending"), + ("event_id", "ascending"), + ] + table = combined.take(pc.sort_indices(combined, sort_keys=sort_keys)) + _validate_bar_rows(table, aggregation, context_before) + if aggregation.kind == "event_bar": + evidence = aggregation.partition_evidence or () + partition_by_path = {item.relative_path: item for item in before.partitions} + evidence_paths = {item.relative_path for item in evidence} + if evidence_paths != set(partition_by_path): + raise ValidationError("event-bar evidence does not cover the Curated partition set") + for item in evidence: + partition = partition_by_path[item.relative_path] + if item.instrument_id != partition.instrument_id: + raise ValidationError("event-bar evidence instrument does not match its partition") + _verify_event_bars( + lake_root, + aggregation, + before.lineage["normalized_snapshot_id"], + table, + context_before, + ) + after = load_curated_snapshot(lake_root, dataset, snapshot_id) + context_after = load_market_context_snapshot(lake_root, aggregation.market_context_snapshot_id) + if before != after: + raise ValidationError("Curated snapshot changed while building verified input") + if context_before.manifest() != context_after.manifest(): + raise ValidationError("market context changed while building verified input") + normalized = load_normalized_snapshot(lake_root, before.lineage["normalized_snapshot_id"]) + if normalized.logical_sha256 != before.lineage["normalized_logical_sha256"]: + raise ValidationError("Curated Normalized lineage hash changed") + return VerifiedFactorInput( + layer="curated", + source_snapshot_id=before.snapshot_id, + source_logical_sha256=before.logical_sha256, + selection_logical_sha256=_table_logical_sha256(table), + event_schemas=(EventSchemaRef(BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + table=table, + calendar_id=aggregation.calendar_id, + session_policy_version=aggregation.session_policy_version, + market_context_snapshot_id=context_before.snapshot_id, + market_context_logical_sha256=context_before.logical_sha256, + lineage=tuple( + sorted( + ( + LineageRef("market", before.snapshot_id, before.logical_sha256), + LineageRef( + "market_context", context_before.snapshot_id, context_before.logical_sha256 + ), + LineageRef("normalized", normalized.snapshot_id, normalized.logical_sha256), + ) + ) + ), + aggregation=aggregation, + ) diff --git a/tests/test_m2_integration.py b/tests/test_m2_integration.py index 2b042a2..c1c001a 100644 --- a/tests/test_m2_integration.py +++ b/tests/test_m2_integration.py @@ -37,11 +37,16 @@ def crypto_context(provider: str) -> qdk.AdapterContext: def test_public_m2_api_and_version_are_exposed() -> None: - assert qdk.__version__ == version("quant-data-kit") == "0.7.4" + assert qdk.__version__ == version("quant-data-kit") == "0.8.0" for name in ( "write_raw_bytes", "write_normalized_events", "curate_trade_bars_from_snapshot", + "curate_session_bars_from_snapshot", + "curate_trade_event_bars_from_snapshot", + "create_market_context_snapshot", + "load_verified_curated_bars", + "load_verified_normalized_events", "DuckDBCatalog", "replay_l2", "BinanceFixtureAdapter", diff --git a/tests/test_m8_research_contracts.py b/tests/test_m8_research_contracts.py new file mode 100644 index 0000000..5486daf --- /dev/null +++ b/tests/test_m8_research_contracts.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +from dataclasses import replace + +import pyarrow as pa +import pytest + +from quant_data_kit.exceptions import ValidationError +from quant_data_kit.fixed_point import FixedPoint +from quant_data_kit.research_contracts_v2 import ( + CURATED_AGGREGATION_SCHEMA_ID, + MARKET_CONTEXT_SCHEMA_ID, + VERIFIED_FACTOR_INPUT_SCHEMA_ID, + CuratedAggregation, + EventBarPartitionEvidence, + EventSchemaRef, + LineageRef, + VerifiedFactorInput, +) +from quant_data_kit.schemas_v2 import ( + BOOK_DELTA_EVENT_SCHEMA_ID, + SCHEMA_VERSION_V2, + TRADE_EVENT_SCHEMA_ID, +) + +HASH = "0" * 64 +OTHER_HASH = "1" * 64 +SNAPSHOT = f"sha256-{HASH}" +TRADE_SCHEMA = EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2) + + +def evidence( + *, + first: int = 1, + last: int = 2, + digest: str = HASH, + path: str = "date=2026-01-05/instrument=IF/data.parquet", +) -> EventBarPartitionEvidence: + return EventBarPartitionEvidence( + relative_path=path, + source="fixture", + instrument_id="IF", + session_id="day", + first_sequence=first, + last_sequence=last, + first_event_id=f"e{first}", + last_event_id=f"e{last}", + event_count=last - first + 1, + source_selection_sha256=digest, + ) + + +def fixed_aggregation() -> CuratedAggregation: + return CuratedAggregation( + calendar_id="calendar-v1", + session_policy_version="sessions-v1", + kind="fixed_time_bar", + recipe_version="recipe-v1", + interval_ns=60_000_000_000, + market_context_snapshot_id=SNAPSHOT, + market_context_logical_sha256=HASH, + source_event_schemas=(TRADE_SCHEMA,), + ) + + +def event_aggregation( + items: tuple[EventBarPartitionEvidence, ...] | None = None, +) -> CuratedAggregation: + return CuratedAggregation( + calendar_id="calendar-v1", + session_policy_version="sessions-v1", + kind="event_bar", + recipe_version="recipe-v1", + event_bar_basis="trade_count", + event_bar_threshold=FixedPoint(2, 0), + market_context_snapshot_id=SNAPSHOT, + market_context_logical_sha256=HASH, + source_event_schemas=(TRADE_SCHEMA,), + partition_evidence=items or (evidence(),), + ) + + +def test_schema_constants_and_round_trips_are_frozen() -> None: + assert CURATED_AGGREGATION_SCHEMA_ID == "puresaber.curated-aggregation@1.0.0" + assert MARKET_CONTEXT_SCHEMA_ID == "puresaber.market-context@1.0.0" + assert VERIFIED_FACTOR_INPUT_SCHEMA_ID == "puresaber.verified-factor-input@1.0.0" + assert EventSchemaRef.from_contract(TRADE_SCHEMA.to_contract()) == TRADE_SCHEMA + aggregation = event_aggregation() + assert CuratedAggregation.from_contract(aggregation.to_contract()) == aggregation + assert EventBarPartitionEvidence.from_contract(evidence().to_contract()) == evidence() + + +@pytest.mark.parametrize( + ("schema_id", "version"), + [("", SCHEMA_VERSION_V2), ("foreign.trade", SCHEMA_VERSION_V2), (TRADE_EVENT_SCHEMA_ID, "v2")], +) +def test_event_schema_reference_rejects_invalid_identity(schema_id: str, version: str) -> None: + with pytest.raises(ValidationError): + EventSchemaRef(schema_id, version) + + +def test_closed_references_and_lineage_fail_closed() -> None: + with pytest.raises(ValidationError, match="exactly"): + EventSchemaRef.from_contract( + {"schema_id": TRADE_EVENT_SCHEMA_ID, "schema_version": SCHEMA_VERSION_V2, "x": 1} + ) + for args in (("", SNAPSHOT, HASH), ("market", "latest", HASH), ("market", SNAPSHOT, "X")): + with pytest.raises(ValidationError): + LineageRef(*args) + + +@pytest.mark.parametrize( + "changes", + [ + {"source": ""}, + {"first_sequence": True}, + {"first_sequence": -1}, + {"event_count": 0}, + {"first_sequence": 3, "last_sequence": 2}, + {"source_selection_sha256": "bad"}, + ], +) +def test_event_evidence_rejects_invalid_values(changes: dict) -> None: + with pytest.raises(ValidationError): + replace(evidence(), **changes) + + +@pytest.mark.parametrize("value", ["-0", "01", str(2**63), 1]) +def test_event_evidence_requires_canonical_integer_strings(value: object) -> None: + payload = evidence().to_contract() + payload["first_sequence"] = value + with pytest.raises(ValidationError, match="canonical|int64"): + EventBarPartitionEvidence.from_contract(payload) + + +def test_aggregation_kind_conditions_and_order_are_strict() -> None: + base = fixed_aggregation() + with pytest.raises(ValidationError, match="unsupported"): + replace(base, kind="other") + with pytest.raises(ValidationError, match="source_event_schemas"): + replace(base, source_event_schemas=()) + with pytest.raises(ValidationError, match="interval_ns"): + replace(base, interval_ns=0) + with pytest.raises(ValidationError, match="another"): + replace(base, session_rollup="session") + session = replace(base, kind="session_bar", interval_ns=None, session_rollup="session") + assert session.session_rollup == "session" + with pytest.raises(ValidationError, match="session_rollup"): + replace(session, session_rollup=None) + with pytest.raises(ValidationError, match="another"): + replace(session, interval_ns=1) + event = event_aggregation() + with pytest.raises(ValidationError, match="basis"): + replace(event, event_bar_basis="bad") + with pytest.raises(ValidationError, match="threshold"): + replace(event, event_bar_threshold=FixedPoint(0, 0)) + with pytest.raises(ValidationError, match="another"): + replace(event, interval_ns=1) + with pytest.raises(ValidationError, match="evidence"): + replace(event, partition_evidence=()) + duplicate = evidence(first=3, last=4) + with pytest.raises(ValidationError, match="globally unique"): + replace(event, partition_evidence=(evidence(), duplicate)) + second = evidence(first=3, last=4, digest=OTHER_HASH) + with pytest.raises(ValidationError, match="sorted"): + replace(event, partition_evidence=(second, evidence())) + overlap = evidence(first=2, last=3, digest=OTHER_HASH) + with pytest.raises(ValidationError, match="overlap"): + replace(event, partition_evidence=(evidence(), overlap)) + + +def test_aggregation_parser_rejects_noncanonical_and_open_payloads() -> None: + payload = fixed_aggregation().to_contract() + payload["extra"] = True + with pytest.raises(ValidationError, match="exactly"): + CuratedAggregation.from_contract(payload) + payload = fixed_aggregation().to_contract() + payload["interval_ns"] = "-0" + with pytest.raises(ValidationError, match="canonical"): + CuratedAggregation.from_contract(payload) + payload = event_aggregation().to_contract() + assert payload["event_bar_threshold"] is not None + payload["event_bar_threshold"]["scale"] = True + with pytest.raises(ValidationError, match="scale"): + CuratedAggregation.from_contract(payload) + payload = fixed_aggregation().to_contract() + payload["source_event_schemas"] = "not-an-array" + with pytest.raises(ValidationError, match="array"): + CuratedAggregation.from_contract(payload) + payload = fixed_aggregation().to_contract() + payload["partition_evidence"] = "not-an-array" + with pytest.raises(ValidationError, match="array"): + CuratedAggregation.from_contract(payload) + + +def valid_verified_input(**changes) -> VerifiedFactorInput: + values = { + "layer": "normalized", + "source_snapshot_id": SNAPSHOT, + "source_logical_sha256": HASH, + "selection_logical_sha256": OTHER_HASH, + "event_schemas": (TRADE_SCHEMA,), + "table": pa.table({"value": [1]}), + "calendar_id": "calendar-v1", + "session_policy_version": "sessions-v1", + "market_context_snapshot_id": SNAPSHOT, + "market_context_logical_sha256": HASH, + "lineage": (LineageRef("market", SNAPSHOT, HASH),), + } + values.update(changes) + return VerifiedFactorInput(**values) + + +def test_verified_input_is_closed_nonempty_and_layer_safe() -> None: + valid = valid_verified_input() + assert len(valid.arrow_schema_sha256) == 64 + assert valid.to_contract()["rows"] == "1" + cases = [ + {"schema_id": "wrong"}, + {"layer": "raw"}, + {"event_schemas": ()}, + {"table": pa.table({"value": pa.array([], type=pa.int64())})}, + {"lineage": ()}, + {"layer": "curated", "aggregation": None}, + {"aggregation": fixed_aggregation()}, + ] + for changes in cases: + with pytest.raises(ValidationError): + valid_verified_input(**changes) + curated = valid_verified_input(layer="curated", aggregation=fixed_aggregation()) + assert curated.to_contract()["aggregation"]["kind"] == "fixed_time_bar" + + +def test_schema_refs_must_be_sorted_when_multiple() -> None: + book = EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2) + with pytest.raises(ValidationError, match="sorted"): + valid_verified_input(event_schemas=(TRADE_SCHEMA, book)) diff --git a/tests/test_m8_research_inputs.py b/tests/test_m8_research_inputs.py new file mode 100644 index 0000000..8922109 --- /dev/null +++ b/tests/test_m8_research_inputs.py @@ -0,0 +1,1033 @@ +from __future__ import annotations + +import json +from copy import copy +from dataclasses import replace +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import pyarrow as pa +import pytest + +import quant_data_kit.curated as curated_module +import quant_data_kit.research_inputs_v2 as research_inputs +from quant_data_kit.curated import ( + build_event_bars, + build_session_rollup_bars, + curate_session_bars_from_snapshot, + curate_trade_bars_from_snapshot, + curate_trade_event_bars_from_snapshot, +) +from quant_data_kit.data_lake import ( + StoragePolicy, + read_normalized_events, + write_normalized_events, + write_raw_bytes, +) +from quant_data_kit.domain_v2 import ( + AssetClass, + InstrumentSpec, + MarginMode, + SessionPhase, + TradingSession, +) +from quant_data_kit.exceptions import ValidationError +from quant_data_kit.fixed_point import FixedPoint +from quant_data_kit.research_contracts_v2 import ( + EventSchemaRef, +) +from quant_data_kit.research_inputs_v2 import ( + create_market_context_snapshot, + load_market_context_snapshot, + load_verified_curated_bars, + load_verified_normalized_events, +) +from quant_data_kit.schemas_v2 import ( + BAR_EVENT_SCHEMA_ID, + BOOK_DELTA_EVENT_SCHEMA_ID, + BOOK_SNAPSHOT_EVENT_SCHEMA_ID, + QUOTE_EVENT_SCHEMA_ID, + SCHEMA_VERSION_V2, + TRADE_EVENT_SCHEMA_ID, + get_arrow_schema, +) + +UTC = timezone.utc +TEST_POLICY = StoragePolicy( + hot_quota_bytes=1024**3, + minimum_free_bytes=1, + minimum_free_fraction=0.000001, +) +SESSION_ID = "CFFEX-IF-2026-01-05-DAY" + + +def trade(event_id: str, timestamp: str, sequence: int, price: int = 40001) -> dict: + return { + "event_type": "trade", + "event_id": event_id, + "instrument_id": "IF-CONT", + "event_time": timestamp, + "received_at": timestamp, + "available_at": timestamp, + "source": "cn-fixture", + "trading_day": "2026-01-05", + "session_id": SESSION_ID, + "sequence": sequence, + "price": {"units": price, "scale": 1}, + "quantity": {"units": 1, "scale": 0}, + "aggressor_side": "unknown", + } + + +def normalized(root: Path, records: list[dict], key: str = "m8"): + raw = write_raw_bytes( + root, + source="cn-fixture", + request={"fixture": key}, + collected_at="2026-01-05T01:00:00Z", + payload=key.encode(), + idempotency_key=key, + policy=TEST_POLICY, + ) + result = write_normalized_events( + root, + records, + provider="cn-fixture", + venue="CFFEX", + upstream_raw_references=[raw.reference()], + policy=TEST_POLICY, + ) + assert result.snapshot is not None + return result.snapshot + + +def market_context(root: Path): + instrument = InstrumentSpec( + instrument_id="IF-CONT", + asset_class=AssetClass.FUTURE, + product_type="index-future", + venue="CFFEX", + native_symbol="IF", + settlement_currency="CNY", + price_tick=FixedPoint(2, 1), + quantity_step=FixedPoint(1, 0), + contract_multiplier=FixedPoint(300, 0), + calendar_id="cffex-v1", + margin_mode=MarginMode.CROSS, + effective_from=datetime(2025, 1, 1, tzinfo=UTC), + available_at=datetime(2025, 1, 1, tzinfo=UTC), + ) + session = TradingSession( + session_id=SESSION_ID, + calendar_id="cffex-v1", + venue="CFFEX", + trading_day=date(2026, 1, 5), + phase=SessionPhase.CONTINUOUS, + opens_at=datetime(2026, 1, 5, 1, 30, tzinfo=UTC), + closes_at=datetime(2026, 1, 5, 2, 0, tzinfo=UTC), + available_at=datetime(2025, 12, 1, tzinfo=UTC), + ) + return create_market_context_snapshot( + root, + calendar_id="cffex-v1", + session_policy_version="cffex-session-v1", + instruments=[instrument], + sessions=[session], + policy=TEST_POLICY, + ) + + +def test_market_context_and_verified_normalized_input_are_content_bound(tmp_path: Path) -> None: + source = normalized( + tmp_path, + [ + trade("t1", "2026-01-05T01:30:01Z", 1), + trade("t2", "2026-01-05T01:30:02Z", 2), + ], + ) + context = market_context(tmp_path) + assert load_market_context_snapshot(tmp_path, context.snapshot_id) == context + verified = load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + assert verified.layer == "normalized" + assert verified.table.num_rows == 2 + assert verified.table.column("event_schema_id").to_pylist() == [ + TRADE_EVENT_SCHEMA_ID, + TRADE_EVENT_SCHEMA_ID, + ] + assert verified.to_contract()["rows"] == "2" + assert verified.to_contract()["aggregation"] is None + + +def test_fixed_session_and_event_bars_all_load_through_certified_factory( + tmp_path: Path, +) -> None: + source = normalized( + tmp_path, + [ + trade("t1", "2026-01-05T01:30:01Z", 1, 40001), + trade("t2", "2026-01-05T01:30:20Z", 2, 40003), + trade("t3", "2026-01-05T01:31:01Z", 3, 40002), + trade("t4", "2026-01-05T01:31:20Z", 4, 40004), + ], + ) + context = market_context(tmp_path) + fixed = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="fixed-bars", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + session = curate_session_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="session-bars", + revision_id="r1", + recipe_version="session-v1", + session_rollup="session", + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + event = curate_trade_event_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="event-bars", + revision_id="r1", + recipe_version="event-v1", + basis="trade_count", + threshold=FixedPoint(2, 0), + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + fixed_input = load_verified_curated_bars(tmp_path, "fixed-bars", fixed.snapshot_id) + session_input = load_verified_curated_bars(tmp_path, "session-bars", session.snapshot_id) + event_input = load_verified_curated_bars(tmp_path, "event-bars", event.snapshot_id) + assert fixed_input.table.num_rows == 2 + assert fixed_input.aggregation is not None + assert fixed_input.aggregation.kind == "fixed_time_bar" + assert session_input.table.num_rows == 1 + assert session_input.aggregation is not None + assert session_input.aggregation.kind == "session_bar" + assert event_input.table.num_rows == 2 + assert event_input.aggregation is not None + assert event_input.aggregation.partition_evidence is not None + assert event_input.to_contract()["event_schemas"] == [ + {"schema_id": BAR_EVENT_SCHEMA_ID, "schema_version": SCHEMA_VERSION_V2} + ] + + +def test_legacy_curated_snapshot_cannot_be_promoted_to_m8(tmp_path: Path) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + legacy = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="legacy", + revision_id="r1", + recipe_version="legacy-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + policy=TEST_POLICY, + ) + with pytest.raises(ValidationError, match="legacy-curated-not-m8-certified"): + load_verified_curated_bars(tmp_path, "legacy", legacy.snapshot_id) + + +def test_closed_schema_and_context_mutation_fail_closed(tmp_path: Path) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + with pytest.raises(ValidationError, match="event_schemas"): + load_verified_normalized_events(tmp_path, source.snapshot_id, [], context.snapshot_id) + with pytest.raises(ValidationError, match="non-Bar"): + load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + with pytest.raises(ValidationError, match="sorted"): + load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [ + EventSchemaRef(BOOK_SNAPSHOT_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + ], + context.snapshot_id, + ) + manifest_path = ( + tmp_path / "market-context" / "snapshots" / context.snapshot_id / "manifest.json" + ) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + payload["session_policy_version"] = "tampered" + manifest_path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValidationError, match="logical hash changed"): + load_market_context_snapshot(tmp_path, context.snapshot_id) + + +def test_market_context_creation_rejects_invalid_members_and_is_idempotent( + tmp_path: Path, +) -> None: + context = market_context(tmp_path) + instrument = context.instruments[0] + session = context.sessions[0] + repeated = create_market_context_snapshot( + tmp_path, + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + instruments=context.instruments, + sessions=context.sessions, + policy=TEST_POLICY, + ) + assert repeated == context + invalid_calls = [ + {"calendar_id": "", "instruments": [instrument], "sessions": [session]}, + { + "calendar_id": context.calendar_id, + "session_policy_version": "", + "instruments": [instrument], + "sessions": [session], + }, + {"calendar_id": context.calendar_id, "instruments": [], "sessions": [session]}, + { + "calendar_id": context.calendar_id, + "instruments": [replace(instrument, calendar_id="other")], + "sessions": [session], + }, + { + "calendar_id": context.calendar_id, + "instruments": [instrument], + "sessions": [replace(session, calendar_id="other")], + }, + { + "calendar_id": context.calendar_id, + "instruments": [instrument, instrument], + "sessions": [session], + }, + { + "calendar_id": context.calendar_id, + "instruments": [instrument], + "sessions": [session, session], + }, + ] + for values in invalid_calls: + values.setdefault("session_policy_version", context.session_policy_version) + with pytest.raises(ValidationError): + create_market_context_snapshot(tmp_path / "invalid", policy=TEST_POLICY, **values) + with pytest.raises(ValidationError, match="InstrumentSpec"): + create_market_context_snapshot( + tmp_path / "wrong-instrument", + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + instruments=[object()], + sessions=[session], + policy=TEST_POLICY, + ) + with pytest.raises(ValidationError, match="TradingSession"): + create_market_context_snapshot( + tmp_path / "wrong-session", + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + instruments=[instrument], + sessions=[object()], + policy=TEST_POLICY, + ) + + +def test_market_context_loader_rejects_paths_shape_and_extra_files(tmp_path: Path) -> None: + context = market_context(tmp_path) + with pytest.raises(ValidationError, match="content-addressed"): + load_market_context_snapshot(tmp_path, "latest") + snapshot_dir = tmp_path / "market-context" / "snapshots" / context.snapshot_id + (snapshot_dir / "unexpected.txt").write_text("x", encoding="utf-8") + with pytest.raises(ValidationError, match="unexpected"): + load_market_context_snapshot(tmp_path, context.snapshot_id) + + malformed_root = tmp_path / "malformed" + malformed = market_context(malformed_root) + malformed_manifest = ( + malformed_root / "market-context" / "snapshots" / malformed.snapshot_id / "manifest.json" + ) + malformed_manifest.write_text("{", encoding="utf-8") + with pytest.raises(ValidationError, match="malformed"): + load_market_context_snapshot(malformed_root, malformed.snapshot_id) + + open_root = tmp_path / "open" + opened = market_context(open_root) + open_manifest = ( + open_root / "market-context" / "snapshots" / opened.snapshot_id / "manifest.json" + ) + payload = json.loads(open_manifest.read_text(encoding="utf-8")) + payload["extra"] = True + open_manifest.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValidationError, match="not closed"): + load_market_context_snapshot(open_root, opened.snapshot_id) + + +def test_context_record_guards_cover_pit_session_and_venue(tmp_path: Path) -> None: + context = market_context(tmp_path) + record = trade("t1", "2026-01-05T01:30:01Z", 1) + with pytest.raises(ValidationError, match="valid UTC"): + research_inputs._utc("not-a-time", "time") + with pytest.raises(ValidationError, match="UTC-aware"): + research_inputs._utc("2026-01-05T01:30:01", "time") + with pytest.raises(ValidationError, match="closed FixedPoint"): + research_inputs._fixed({"units": 1, "scale": 0, "x": 1}, "value") + with pytest.raises(ValidationError, match="no PIT-valid"): + research_inputs._active_instrument( + context, + "missing", + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + ) + duplicate = replace(context.instruments[0], available_at=datetime(2025, 2, 1, tzinfo=UTC)) + ambiguous = replace(context, instruments=(context.instruments[0], duplicate)) + with pytest.raises(ValidationError, match="ambiguous"): + research_inputs._active_instrument( + ambiguous, + "IF-CONT", + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + ) + with pytest.raises(ValidationError, match="exactly one"): + research_inputs._context_session( + context, + "missing", + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + datetime(2026, 1, 5, 1, 30, 1, tzinfo=UTC), + date(2026, 1, 5), + allow_close=False, + ) + outside = dict( + record, + event_time="2026-01-05T02:00:00Z", + received_at="2026-01-05T02:00:00Z", + available_at="2026-01-05T02:00:00Z", + ) + with pytest.raises(ValidationError, match="outside session"): + research_inputs._validate_context_record(context, outside, bar=False) + wrong_day = dict(record, trading_day="2026-01-06") + with pytest.raises(ValidationError, match="trading_day"): + research_inputs._validate_context_record(context, wrong_day, bar=False) + future_session = replace( + context, + sessions=( + replace(context.sessions[0], available_at=datetime(2026, 1, 5, 1, 31, tzinfo=UTC)), + ), + ) + with pytest.raises(ValidationError, match="not PIT-valid"): + research_inputs._validate_context_record(future_session, record, bar=False) + bad_pit = dict(record, received_at="2026-01-05T01:30:00Z") + with pytest.raises(ValidationError, match="not monotonic"): + research_inputs._validate_context_record(context, bad_pit, bar=False) + wrong_calendar = replace(context, calendar_id="other") + with pytest.raises(ValidationError, match="calendar"): + research_inputs._validate_context_record(wrong_calendar, record, bar=False) + wrong_venue = replace(context, instruments=(replace(context.instruments[0], venue="OTHER"),)) + with pytest.raises(ValidationError, match="venue"): + research_inputs._validate_context_record(wrong_venue, record, bar=False) + + +def l2_event(event_type: str, event_id: str, sequence: int, timestamp: str) -> dict: + common = { + "event_type": event_type, + "event_id": event_id, + "instrument_id": "BTC-USDT", + "event_time": timestamp, + "received_at": timestamp, + "available_at": timestamp, + "source": "fixture", + "trading_day": "2026-01-05", + "session_id": "crypto-day", + "sequence": sequence, + } + if event_type == "book_snapshot": + return { + **common, + "bids": [ + { + "price": {"units": 100, "scale": 0}, + "quantity": {"units": 1, "scale": 0}, + "order_count": 1, + } + ], + "asks": [ + { + "price": {"units": 101, "scale": 0}, + "quantity": {"units": 1, "scale": 0}, + "order_count": 1, + } + ], + } + return { + **common, + "side": "bid", + "action": "upsert", + "price": {"units": 99, "scale": 0}, + "quantity": {"units": 2, "scale": 0}, + "previous_sequence": sequence - 1, + } + + +def test_event_order_union_and_schema_guards() -> None: + trade_ref = EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2) + first = trade("same", "2026-01-05T01:30:01Z", 2) + duplicate = trade("same", "2026-01-05T01:30:02Z", 3) + with pytest.raises(ValidationError, match="duplicate event_id"): + research_inputs._validate_event_order([(trade_ref, first), (trade_ref, duplicate)]) + earlier = trade("earlier", "2026-01-05T01:30:00Z", 3) + with pytest.raises(ValidationError, match="not strictly ordered"): + research_inputs._validate_event_order([(trade_ref, first), (trade_ref, earlier)]) + lower_sequence = trade("lower", "2026-01-05T01:30:02Z", 1) + with pytest.raises(ValidationError, match="sequence does not advance"): + research_inputs._validate_event_order([(trade_ref, first), (trade_ref, lower_sequence)]) + + snapshot_ref = EventSchemaRef(BOOK_SNAPSHOT_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2) + delta_ref = EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2) + snapshot = l2_event("book_snapshot", "s1", 10, "2026-01-05T01:30:01Z") + delta = l2_event("book_delta", "d1", 11, "2026-01-05T01:30:02Z") + research_inputs._validate_event_order([(snapshot_ref, snapshot), (delta_ref, delta)]) + union = research_inputs._event_union_table([(snapshot_ref, snapshot), (delta_ref, delta)]) + assert union.num_rows == 2 + assert union.schema.field("bids").nullable + with pytest.raises(ValidationError, match="failed replay"): + research_inputs._validate_event_order([(delta_ref, delta)]) + with pytest.raises(ValidationError, match="version 2.0.0"): + research_inputs._normalize_schema_refs([EventSchemaRef(TRADE_EVENT_SCHEMA_ID, "1.0.0")]) + assert research_inputs._normalize_schema_refs( + [{"schema_id": TRADE_EVENT_SCHEMA_ID, "schema_version": SCHEMA_VERSION_V2}] + ) == (trade_ref,) + + +def test_normalized_factory_missing_schema_and_toctou_guards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + refs = [ + EventSchemaRef(QUOTE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + ] + with pytest.raises(ValidationError, match="lacks requested"): + load_verified_normalized_events(tmp_path, source.snapshot_id, refs, context.snapshot_id) + + original = research_inputs.load_normalized_snapshot + calls = 0 + + def changed_snapshot(*args, **kwargs): + nonlocal calls + calls += 1 + result = original(*args, **kwargs) + return replace(result, logical_sha256="f" * 64) if calls == 2 else result + + monkeypatch.setattr(research_inputs, "load_normalized_snapshot", changed_snapshot) + with pytest.raises(ValidationError, match="changed while"): + load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + + +def test_normalized_factory_context_toctou_guard( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + original = research_inputs.load_market_context_snapshot + calls = 0 + + def changed_context(*args, **kwargs): + nonlocal calls + calls += 1 + result = original(*args, **kwargs) + return replace(result, session_policy_version="changed") if calls == 2 else result + + monkeypatch.setattr(research_inputs, "load_market_context_snapshot", changed_context) + with pytest.raises(ValidationError, match="context changed"): + load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + + +def test_payload_parsers_and_arrow_null_conversion_are_closed(tmp_path: Path) -> None: + context = market_context(tmp_path) + instrument_payload = context.identity()["instruments"][0] + with pytest.raises(ValidationError, match="instrument context record"): + research_inputs._instrument_from_payload({}) + bad_metadata = dict(instrument_payload, metadata=[]) + with pytest.raises(ValidationError, match="metadata"): + research_inputs._instrument_from_payload(bad_metadata) + with pytest.raises(ValidationError, match="session context record"): + research_inputs._session_from_payload({}) + nullable_schema = pa.schema([pa.field("missing", pa.timestamp("ns", tz="UTC"), nullable=True)]) + assert research_inputs._arrow_ready({}, nullable_schema) == {} + + +def test_bar_validation_and_event_evidence_fail_closed(tmp_path: Path) -> None: + source = normalized( + tmp_path, + [ + trade("t1", "2026-01-05T01:30:01Z", 1, 40001), + trade("t2", "2026-01-05T01:30:20Z", 2, 40003), + trade("t3", "2026-01-05T01:31:01Z", 3, 40002), + trade("t4", "2026-01-05T01:31:20Z", 4, 40004), + ], + ) + context_snapshot = market_context(tmp_path) + fixed = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="guard-fixed", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context_snapshot.snapshot_id, + policy=TEST_POLICY, + ) + event = curate_trade_event_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="guard-event", + revision_id="r1", + recipe_version="event-v1", + basis="trade_count", + threshold=FixedPoint(2, 0), + market_context_snapshot_id=context_snapshot.snapshot_id, + policy=TEST_POLICY, + ) + fixed_input = load_verified_curated_bars(tmp_path, "guard-fixed", fixed.snapshot_id) + event_input = load_verified_curated_bars(tmp_path, "guard-event", event.snapshot_id) + context = load_market_context_snapshot(tmp_path, context_snapshot.snapshot_id) + assert fixed_input.aggregation is not None + assert event_input.aggregation is not None + + def bars(rows: list[dict]) -> pa.Table: + return pa.Table.from_pylist(rows, schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID)) + + fixed_rows = fixed_input.table.to_pylist() + incomplete = [dict(fixed_rows[0], is_complete=False)] + with pytest.raises(ValidationError, match="incomplete"): + research_inputs._validate_bar_rows(bars(incomplete), fixed_input.aggregation, context) + bad_boundary = [dict(fixed_rows[0], event_time=fixed_rows[0]["bar_start"])] + with pytest.raises(ValidationError, match="event_time"): + research_inputs._validate_bar_rows(bars(bad_boundary), fixed_input.aggregation, context) + with pytest.raises(ValidationError, match="duplicate"): + research_inputs._validate_bar_rows( + bars([fixed_rows[0], fixed_rows[0]]), fixed_input.aggregation, context + ) + with pytest.raises(ValidationError, match="strictly ordered"): + research_inputs._validate_bar_rows( + bars(list(reversed(fixed_rows))), fixed_input.aggregation, context + ) + wrong_interval = replace(fixed_input.aggregation, interval_ns=1) + with pytest.raises(ValidationError, match="interval"): + research_inputs._validate_bar_rows(bars([fixed_rows[0]]), wrong_interval, context) + crossing_row = dict( + fixed_rows[0], + bar_start=datetime(2026, 1, 5, 1, 29, tzinfo=UTC), + bar_end=datetime(2026, 1, 5, 1, 30, tzinfo=UTC), + event_time=datetime(2026, 1, 5, 1, 30, tzinfo=UTC), + ) + with pytest.raises(ValidationError, match="crosses"): + research_inputs._validate_bar_rows(bars([crossing_row]), fixed_input.aggregation, context) + + source_rows = read_normalized_events(tmp_path, source.snapshot_id, event_type="trade") + evidence = event_input.aggregation.partition_evidence + assert evidence is not None + with pytest.raises(ValidationError, match="event_count"): + research_inputs._source_rows_for_evidence( + source_rows, replace(evidence[0], event_count=evidence[0].event_count + 1) + ) + with pytest.raises(ValidationError, match="boundaries"): + research_inputs._source_rows_for_evidence( + source_rows, replace(evidence[0], first_event_id="wrong") + ) + with pytest.raises(ValidationError, match="selection hash"): + research_inputs._source_rows_for_evidence( + source_rows, replace(evidence[0], source_selection_sha256="f" * 64) + ) + + broken = copy(event_input.aggregation) + object.__setattr__(broken, "partition_evidence", None) + with pytest.raises(ValidationError, match="metadata is incomplete"): + research_inputs._verify_event_bars( + tmp_path, broken, source.snapshot_id, event_input.table, context + ) + wrong_schema = replace( + event_input.aggregation, + source_event_schemas=(EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + ) + with pytest.raises(ValidationError, match="Trade schema"): + research_inputs._verify_event_bars( + tmp_path, wrong_schema, source.snapshot_id, event_input.table, context + ) + multiple_sources = event_input.table.to_pylist() + multiple_sources[1] = dict(multiple_sources[1], source="other") + with pytest.raises(ValidationError, match="one deterministic"): + research_inputs._verify_event_bars( + tmp_path, + event_input.aggregation, + source.snapshot_id, + bars(multiple_sources), + context, + ) + changed = event_input.table.to_pylist() + changed[0] = dict(changed[0]) + changed[0]["close_price"] = dict(changed[0]["close_price"], units=40002) + with pytest.raises(ValidationError, match="do not recompute"): + research_inputs._verify_event_bars( + tmp_path, event_input.aggregation, source.snapshot_id, bars(changed), context + ) + + +def test_curated_factory_binding_and_toctou_guards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + curated = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="toctou-bars", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + original_curated = research_inputs.load_curated_snapshot + calls = 0 + + def changed_curated(*args, **kwargs): + nonlocal calls + calls += 1 + result = original_curated(*args, **kwargs) + return replace(result, created_at="2026-01-05T01:59:00Z") if calls == 2 else result + + monkeypatch.setattr(research_inputs, "load_curated_snapshot", changed_curated) + with pytest.raises(ValidationError, match="Curated snapshot changed"): + load_verified_curated_bars(tmp_path, "toctou-bars", curated.snapshot_id) + + +def test_curated_factory_context_and_lineage_guards( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + curated = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="binding-bars", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + original_context = research_inputs.load_market_context_snapshot + + def wrong_context(*args, **kwargs): + return replace(original_context(*args, **kwargs), logical_sha256="f" * 64) + + monkeypatch.setattr(research_inputs, "load_market_context_snapshot", wrong_context) + with pytest.raises(ValidationError, match="binding changed"): + load_verified_curated_bars(tmp_path, "binding-bars", curated.snapshot_id) + + +def test_curated_factory_rechecks_normalized_lineage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + curated = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="lineage-bars", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + original = research_inputs.load_normalized_snapshot + + def changed_normalized(*args, **kwargs): + return replace(original(*args, **kwargs), logical_sha256="f" * 64) + + monkeypatch.setattr(research_inputs, "load_normalized_snapshot", changed_normalized) + with pytest.raises(ValidationError, match="lineage hash changed"): + load_verified_curated_bars(tmp_path, "lineage-bars", curated.snapshot_id) + + +def test_curated_aggregation_builders_cover_all_kinds_and_reject_bad_inputs() -> None: + start = datetime(2026, 1, 5, 1, 30, tzinfo=UTC) + end = datetime(2026, 1, 5, 2, 0, tzinfo=UTC) + records = [ + trade("t1", "2026-01-05T01:30:01Z", 1, 40001), + trade("t2", "2026-01-05T01:30:20Z", 2, 40003), + ] + with pytest.raises(ValidationError, match="no trades"): + curated_module._bar_from_trade_group( + [], + bar_start=start, + bar_end=end, + source="curated", + recipe_version="r1", + identity_extra={}, + ) + with pytest.raises(ValidationError, match="strictly positive"): + curated_module._bar_from_trade_group( + records, + bar_start=end, + bar_end=start, + source="curated", + recipe_version="r1", + identity_extra={}, + ) + with pytest.raises(ValidationError, match="session_rollup"): + build_session_rollup_bars( + records, session_boundaries={SESSION_ID: (start, end)}, session_rollup="bad" + ) + with pytest.raises(ValidationError, match="Missing authoritative"): + build_session_rollup_bars(records, session_boundaries={}, session_rollup="session") + with pytest.raises(ValidationError, match="trading-day boundary"): + build_session_rollup_bars( + records, + session_boundaries={SESSION_ID: (start, end)}, + session_rollup="trading_day", + ) + day_bars = build_session_rollup_bars( + records, + session_boundaries={SESSION_ID: (start, end)}, + session_rollup="trading_day", + trading_day_boundaries={ + ("CFFEX", "2026-01-05"): (start, end, SESSION_ID), + }, + instrument_venues={"IF-CONT": "CFFEX"}, + ) + assert len(day_bars) == 1 + assert day_bars[0]["bar_start"] == "2026-01-05T01:30:00Z" + with pytest.raises(ValidationError, match="unsupported"): + build_event_bars( + records, + basis="bad", + threshold=FixedPoint(1, 0), + session_starts={SESSION_ID: start}, + ) + with pytest.raises(ValidationError, match="positive"): + build_event_bars( + records, + basis="trade_count", + threshold=FixedPoint(0, 0), + session_starts={SESSION_ID: start}, + ) + with pytest.raises(ValidationError, match="scale zero"): + build_event_bars( + records, + basis="trade_count", + threshold=FixedPoint(10, 1), + session_starts={SESSION_ID: start}, + ) + with pytest.raises(ValidationError, match="Missing session start"): + build_event_bars( + records, + basis="trade_count", + threshold=FixedPoint(1, 0), + session_starts={}, + ) + with pytest.raises(ValidationError, match="not strictly ordered"): + build_event_bars( + [records[0], records[0]], + basis="trade_count", + threshold=FixedPoint(2, 0), + session_starts={SESSION_ID: start}, + ) + assert ( + len( + build_event_bars( + records, + basis="base_volume", + threshold=FixedPoint(1, 0), + session_starts={SESSION_ID: start}, + ) + ) + == 2 + ) + assert ( + len( + build_event_bars( + records, + basis="quote_notional", + threshold=FixedPoint(40001, 1), + session_starts={SESSION_ID: start}, + ) + ) + == 2 + ) + with pytest.raises(ValidationError, match="below threshold"): + build_event_bars( + records, + basis="trade_count", + threshold=FixedPoint(3, 0), + session_starts={SESSION_ID: start}, + ) + + +def test_trading_day_rollup_uses_all_authoritative_sessions(tmp_path: Path) -> None: + morning_id = "CFFEX-IF-2026-01-05-AM" + afternoon_id = "CFFEX-IF-2026-01-05-PM" + source_records = [ + dict( + trade("t1", "2026-01-05T01:31:00Z", 1), + session_id=morning_id, + ), + dict( + trade("t2", "2026-01-05T03:01:00Z", 1), + session_id=afternoon_id, + ), + ] + source = normalized(tmp_path, source_records, key="two-sessions") + base_context = market_context(tmp_path) + instrument = base_context.instruments[0] + sessions = [ + TradingSession( + session_id=morning_id, + calendar_id="cffex-v1", + venue="CFFEX", + trading_day=date(2026, 1, 5), + phase=SessionPhase.CONTINUOUS, + opens_at=datetime(2026, 1, 5, 1, 30, tzinfo=UTC), + closes_at=datetime(2026, 1, 5, 2, 0, tzinfo=UTC), + available_at=datetime(2025, 12, 1, tzinfo=UTC), + ), + TradingSession( + session_id=afternoon_id, + calendar_id="cffex-v1", + venue="CFFEX", + trading_day=date(2026, 1, 5), + phase=SessionPhase.CONTINUOUS, + opens_at=datetime(2026, 1, 5, 3, 0, tzinfo=UTC), + closes_at=datetime(2026, 1, 5, 4, 0, tzinfo=UTC), + available_at=datetime(2025, 12, 1, tzinfo=UTC), + ), + ] + context = create_market_context_snapshot( + tmp_path, + calendar_id="cffex-v1", + session_policy_version="cffex-split-v1", + instruments=[instrument], + sessions=sessions, + policy=TEST_POLICY, + ) + curated = curate_session_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="trading-day-bars", + revision_id="r1", + recipe_version="trading-day-v1", + session_rollup="trading_day", + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + verified = load_verified_curated_bars(tmp_path, "trading-day-bars", curated.snapshot_id) + row = verified.table.to_pylist()[0] + assert row["bar_start"] == datetime(2026, 1, 5, 1, 30, tzinfo=UTC) + assert row["bar_end"] == datetime(2026, 1, 5, 4, 0, tzinfo=UTC) + assert row["session_id"] == afternoon_id + + +def test_verified_normalized_l2_requires_snapshot_anchor_and_replays(tmp_path: Path) -> None: + events = [ + dict( + l2_event("book_snapshot", "s1", 10, "2026-01-05T01:30:01Z"), + source="crypto-fixture", + ), + dict( + l2_event("book_delta", "d1", 11, "2026-01-05T01:30:02Z"), + source="crypto-fixture", + ), + ] + raw = write_raw_bytes( + tmp_path, + source="crypto-fixture", + request={"fixture": "l2"}, + collected_at="2026-01-05T01:00:00Z", + payload=b"l2", + idempotency_key="l2", + policy=TEST_POLICY, + ) + result = write_normalized_events( + tmp_path, + events, + provider="crypto-fixture", + venue="BINANCE", + upstream_raw_references=[raw.reference()], + policy=TEST_POLICY, + ) + assert result.snapshot is not None + instrument = InstrumentSpec( + instrument_id="BTC-USDT", + asset_class=AssetClass.CRYPTO, + product_type="spot", + venue="BINANCE", + native_symbol="BTCUSDT", + base_currency="BTC", + quote_currency="USDT", + settlement_currency="USDT", + price_tick=FixedPoint(1, 2), + quantity_step=FixedPoint(1, 6), + contract_multiplier=FixedPoint(1, 0), + calendar_id="crypto-24x7-v1", + margin_mode=MarginMode.CASH, + effective_from=datetime(2025, 1, 1, tzinfo=UTC), + available_at=datetime(2025, 1, 1, tzinfo=UTC), + ) + session = TradingSession( + session_id="crypto-day", + calendar_id="crypto-24x7-v1", + venue="BINANCE", + trading_day=date(2026, 1, 5), + phase=SessionPhase.CONTINUOUS, + opens_at=datetime(2026, 1, 5, tzinfo=UTC), + closes_at=datetime(2026, 1, 6, tzinfo=UTC), + available_at=datetime(2025, 12, 1, tzinfo=UTC), + ) + context = create_market_context_snapshot( + tmp_path, + calendar_id="crypto-24x7-v1", + session_policy_version="utc-day-v1", + instruments=[instrument], + sessions=[session], + policy=TEST_POLICY, + ) + verified = load_verified_normalized_events( + tmp_path, + result.snapshot.snapshot_id, + [ + EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + EventSchemaRef(BOOK_SNAPSHOT_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), + ], + context.snapshot_id, + ) + assert verified.table.num_rows == 2 + assert set(verified.table.column("event_schema_id").to_pylist()) == { + BOOK_DELTA_EVENT_SCHEMA_ID, + BOOK_SNAPSHOT_EVENT_SCHEMA_ID, + } diff --git a/tools/check_branch_coverage.py b/tools/check_branch_coverage.py index c5b8198..290b5f9 100644 --- a/tools/check_branch_coverage.py +++ b/tools/check_branch_coverage.py @@ -11,6 +11,8 @@ "src/quant_data_kit/normalized_v3.py": 90, "src/quant_data_kit/data_lake.py": 90, "src/quant_data_kit/curated.py": 90, + "src/quant_data_kit/research_contracts_v2.py": 90, + "src/quant_data_kit/research_inputs_v2.py": 90, "src/quant_data_kit/process_lock.py": 90, "src/quant_data_kit/schemas_v2.py": 90, "src/quant_data_kit/l2_replay.py": 90, From 7ae3a2f8174155620105a4391e502cffb3e846a1 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 11:19:55 +0800 Subject: [PATCH 2/4] fix: close M8 verified input integrity gaps --- src/quant_data_kit/curated.py | 96 ++++++- src/quant_data_kit/research_contracts_v2.py | 69 ++++- src/quant_data_kit/research_inputs_v2.py | 218 +++++++++++--- tests/test_m8_research_contracts.py | 115 +++++++- tests/test_m8_research_inputs.py | 304 +++++++++++++++++++- 5 files changed, 735 insertions(+), 67 deletions(-) diff --git a/src/quant_data_kit/curated.py b/src/quant_data_kit/curated.py index 631a21a..f3bf742 100644 --- a/src/quant_data_kit/curated.py +++ b/src/quant_data_kit/curated.py @@ -8,7 +8,7 @@ from collections import defaultdict from collections.abc import Iterable, Mapping from dataclasses import asdict, dataclass -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from decimal import Decimal from pathlib import Path from typing import Any, Protocol @@ -671,10 +671,14 @@ def _write_curated_bars( records = [dict(item) for item in bars] if not records: raise ValidationError("Cannot write an empty Curated snapshot") - groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + event_partitioned = aggregation is not None and aggregation.kind == "event_bar" + groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) for record in records: validate_json_record(BAR_EVENT_SCHEMA_ID, record) - groups[(str(record["trading_day"]), str(record["instrument_id"]))].append(record) + key = (str(record["trading_day"]), str(record["instrument_id"])) + if event_partitioned: + key += (str(record["session_id"]),) + groups[key].append(record) estimated_bytes = sum(len(_canonical(_json_value(item))) for item in records) curated_root = _mkdir_in_lake(lake_root, lake_root / "curated" / dataset) @@ -692,7 +696,8 @@ def _write_curated_bars( projected_write_bytes=estimated_bytes, policy=policy, ) - for (trading_date, instrument_id), group in sorted(groups.items()): + for key, group in sorted(groups.items()): + trading_date, instrument_id = key[:2] ordered = sorted( group, key=lambda row: (row["event_time"], int(row["sequence"]), row["event_id"]), @@ -702,9 +707,10 @@ def _write_curated_bars( schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID), ) validate_arrow_table(BAR_EVENT_SCHEMA_ID, table) - relative = Path( - f"date={trading_date}/instrument={quote(instrument_id, safe='-._')}/data.parquet" - ) + partition_root = f"date={trading_date}/instrument={quote(instrument_id, safe='-._')}" + if event_partitioned: + partition_root += f"/session={quote(key[2], safe='-._')}" + relative = Path(f"{partition_root}/data.parquet") target = stage / relative _mkdir_in_lake(lake_root, target.parent) pq.write_table(table, target, compression="zstd", use_dictionary=False) @@ -924,6 +930,9 @@ def curate_trade_event_bars_from_snapshot( ] if not trades: raise ValidationError("Normalized snapshot contains no trades to curate") + upstream_sources = {str(item["source"]) for item in trades} + if len(upstream_sources) != 1: + raise ValidationError("certified event Bars require one upstream source per snapshot") session_starts = {item.session_id: item.opens_at for item in context.sessions} bars = build_event_bars( trades, @@ -955,7 +964,8 @@ def curate_trade_event_bars_from_snapshot( ), ) relative_path = Path( - f"date={trading_day}/instrument={quote(instrument_id, safe='-._')}/data.parquet" + f"date={trading_day}/instrument={quote(instrument_id, safe='-._')}/" + f"session={quote(session_id, safe='-._')}/data.parquet" ).as_posix() evidence.append( EventBarPartitionEvidence( @@ -1006,6 +1016,43 @@ def curate_trade_event_bars_from_snapshot( ) +def _validate_curated_partition_table( + partition: CuratedPartition, + table: pa.Table, + aggregation: CuratedAggregation | None, +) -> str | None: + """Bind partition metadata, source order and event-bar scope to actual rows.""" + validate_arrow_table(partition.schema_id, table) + if table.num_rows != partition.rows: + raise ValidationError("Curated partition row count changed") + rows = table.to_pylist() + previous: tuple[datetime, int, str] | None = None + session_ids: set[str] = set() + for row in rows: + trading_day = row["trading_day"] + trading_day_text = ( + trading_day.isoformat() if isinstance(trading_day, date) else str(trading_day) + ) + if str(row["instrument_id"]) != partition.instrument_id: + raise ValidationError("Curated row instrument does not match partition metadata") + if trading_day_text != partition.trading_date: + raise ValidationError("Curated row trading_day does not match partition metadata") + identity = ( + _utc(row["event_time"], "event_time"), + int(row["sequence"]), + str(row["event_id"]), + ) + if previous is not None and identity <= previous: + raise ValidationError("Curated partition rows are not strictly ordered") + previous = identity + session_ids.add(str(row["session_id"])) + if aggregation is not None and aggregation.kind == "event_bar": + if len(session_ids) != 1: + raise ValidationError("event-bar partition must contain exactly one session") + return next(iter(session_ids)) + return None + + def _load_curated_snapshot( root: Path, dataset: str, @@ -1070,16 +1117,33 @@ def _load_curated_snapshot( rows = 0 expected_files = {Path("manifest.json")} seen_paths: set[str] = set() + event_evidence_by_path = { + item.relative_path: item + for item in ( + snapshot.aggregation.partition_evidence + if snapshot.aggregation is not None + and snapshot.aggregation.kind == "event_bar" + and snapshot.aggregation.partition_evidence is not None + else () + ) + } for partition in snapshot.partitions: if partition.relative_path in seen_paths: raise ValidationError("Curated snapshot contains duplicate partition paths") seen_paths.add(partition.relative_path) if partition.schema_id != BAR_EVENT_SCHEMA_ID: raise ValidationError("Curated partition schema is not the frozen Bar schema") - expected_relative = Path( - f"date={partition.trading_date}/" - f"instrument={quote(partition.instrument_id, safe='-._')}/data.parquet" - ).as_posix() + partition_root = ( + f"date={partition.trading_date}/instrument={quote(partition.instrument_id, safe='-._')}" + ) + expected_event_session: str | None = None + if snapshot.aggregation is not None and snapshot.aggregation.kind == "event_bar": + evidence = event_evidence_by_path.get(partition.relative_path) + if evidence is None or evidence.instrument_id != partition.instrument_id: + raise ValidationError("Curated partition path metadata mismatch") + expected_event_session = evidence.session_id + partition_root += f"/session={quote(expected_event_session, safe='-._')}" + expected_relative = Path(f"{partition_root}/data.parquet").as_posix() if partition.relative_path != expected_relative: raise ValidationError("Curated partition path metadata mismatch") relative = Path(partition.relative_path) @@ -1092,13 +1156,15 @@ def _load_curated_snapshot( if _hash_file(path) != partition.content_sha256: raise ValidationError(f"Curated partition hash changed: {path}") table = pq.ParquetFile(path).read() - validate_arrow_table(partition.schema_id, table) - if table.num_rows != partition.rows: - raise ValidationError(f"Curated partition row count changed: {path}") + event_session_id = _validate_curated_partition_table(partition, table, snapshot.aggregation) + if event_session_id != expected_event_session: + raise ValidationError("event-bar evidence session differs from partition rows") logical_rows = [_json_value(item) for item in table.to_pylist()] if _hash_bytes(_canonical(logical_rows)) != partition.logical_sha256: raise ValidationError(f"Curated partition logical content changed: {path}") rows += table.num_rows + if event_evidence_by_path and set(event_evidence_by_path) != seen_paths: + raise ValidationError("event-bar evidence does not cover Curated partitions") if rows != snapshot.rows: raise ValidationError("Curated snapshot row count changed") actual_files = { diff --git a/src/quant_data_kit/research_contracts_v2.py b/src/quant_data_kit/research_contracts_v2.py index 8ac034b..8a5bc66 100644 --- a/src/quant_data_kit/research_contracts_v2.py +++ b/src/quant_data_kit/research_contracts_v2.py @@ -2,15 +2,22 @@ from __future__ import annotations +import hashlib import re from collections.abc import Mapping from dataclasses import dataclass, field from typing import Any, Literal import pyarrow as pa +from pyarrow import ipc from quant_data_kit.exceptions import ValidationError from quant_data_kit.fixed_point import FixedPoint +from quant_data_kit.schemas_v2 import ( + BAR_EVENT_SCHEMA_ID, + SCHEMA_VERSION_V2, + get_arrow_schema, +) VERIFIED_FACTOR_INPUT_SCHEMA_ID = "puresaber.verified-factor-input@1.0.0" CURATED_AGGREGATION_SCHEMA_ID = "puresaber.curated-aggregation@1.0.0" @@ -23,6 +30,7 @@ _CANONICAL_NONNEGATIVE = re.compile(r"^(?:0|[1-9][0-9]*)$") _CANONICAL_POSITIVE = re.compile(r"^[1-9][0-9]*$") _INT64_MAX = 2**63 - 1 +_VERIFIED_INPUT_FACTORY_TOKEN = object() def _required_text(value: str, field_name: str) -> str: @@ -254,6 +262,9 @@ def __post_init__(self) -> None: evidence = tuple(self.partition_evidence or ()) if not evidence: raise ValidationError("event_bar requires partition evidence") + relative_paths = [item.relative_path for item in evidence] + if len(relative_paths) != len(set(relative_paths)): + raise ValidationError("event-bar partition paths must be globally unique") selection_hashes = [item.source_selection_sha256 for item in evidence] if len(selection_hashes) != len(set(selection_hashes)): raise ValidationError("event-bar selection hashes must be globally unique") @@ -272,6 +283,12 @@ def __post_init__(self) -> None: raise ValidationError("event-bar partition evidence must be canonically sorted") previous_by_stream: dict[tuple[str, str, str], EventBarPartitionEvidence] = {} for item in evidence: + if item.event_count > item.last_sequence - item.first_sequence + 1: + raise ValidationError("event-bar event_count exceeds its sequence range") + if (item.first_sequence == item.last_sequence) != ( + item.first_event_id == item.last_event_id + ): + raise ValidationError("event-bar boundary identities are inconsistent") previous = previous_by_stream.get(item.stream_key) if previous is not None and item.first_sequence <= previous.last_sequence: raise ValidationError("event-bar evidence ranges overlap within one stream") @@ -379,8 +396,11 @@ class VerifiedFactorInput: lineage: tuple[LineageRef, ...] = () aggregation: CuratedAggregation | None = None schema_id: str = VERIFIED_FACTOR_INPUT_SCHEMA_ID + _factory_token: object = field(default=None, repr=False, compare=False) def __post_init__(self) -> None: + if self._factory_token is not _VERIFIED_INPUT_FACTORY_TOKEN: + raise ValidationError("VerifiedFactorInput can only be created by a certified factory") if self.schema_id != VERIFIED_FACTOR_INPUT_SCHEMA_ID: raise ValidationError("unsupported VerifiedFactorInput schema") if self.layer not in {"curated", "normalized"}: @@ -397,21 +417,58 @@ def __post_init__(self) -> None: raise ValidationError("event_schemas must be non-empty, unique, and sorted") if not isinstance(self.table, pa.Table) or self.table.num_rows <= 0: raise ValidationError("verified input table must be a non-empty Arrow table") + if self.selection_logical_sha256 != _arrow_table_logical_sha256(self.table): + raise ValidationError("verified input selection hash does not match its Arrow table") lineage = tuple(self.lineage) if not lineage or lineage != tuple(sorted(lineage)): raise ValidationError("lineage must be non-empty and canonically ordered") + lineage_keys = [(item.role, item.snapshot_id) for item in lineage] + if len(lineage_keys) != len(set(lineage_keys)): + raise ValidationError("lineage roles and snapshots must be unique") if self.layer == "curated": if self.aggregation is None: raise ValidationError("Curated verified input requires aggregation metadata") + if schemas != (EventSchemaRef(BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),): + raise ValidationError("Curated verified input requires the frozen Bar schema") + if self.table.schema != get_arrow_schema(BAR_EVENT_SCHEMA_ID): + raise ValidationError("Curated verified input table is not the frozen Bar schema") + context_values = ( + self.calendar_id, + self.session_policy_version, + self.market_context_snapshot_id, + self.market_context_logical_sha256, + ) + aggregation_values = ( + self.aggregation.calendar_id, + self.aggregation.session_policy_version, + self.aggregation.market_context_snapshot_id, + self.aggregation.market_context_logical_sha256, + ) + if context_values != aggregation_values: + raise ValidationError("verified input context differs from its aggregation") elif self.aggregation is not None: raise ValidationError("Normalized verified input cannot contain aggregation metadata") + elif any( + item.schema_id == BAR_EVENT_SCHEMA_ID or item.schema_version != SCHEMA_VERSION_V2 + for item in schemas + ): + raise ValidationError("Normalized verified input requires non-Bar v2 event schemas") + if self.layer == "normalized": + if "event_schema_id" not in self.table.column_names: + raise ValidationError("Normalized verified input lacks event_schema_id") + actual_schema_ids = set(self.table.column("event_schema_id").to_pylist()) + expected_schema_ids = {item.schema_id for item in schemas} + if actual_schema_ids != expected_schema_ids: + raise ValidationError("Normalized table event schemas differ from its contract") object.__setattr__(self, "event_schemas", schemas) object.__setattr__(self, "lineage", lineage) + @classmethod + def _from_certified_factory(cls, **values: Any) -> VerifiedFactorInput: + return cls(_factory_token=_VERIFIED_INPUT_FACTORY_TOKEN, **values) + @property def arrow_schema_sha256(self) -> str: - import hashlib - return hashlib.sha256(self.table.schema.serialize().to_pybytes()).hexdigest() def to_contract(self) -> dict[str, Any]: @@ -431,3 +488,11 @@ def to_contract(self) -> dict[str, Any]: "arrow_schema_sha256": self.arrow_schema_sha256, "aggregation": self.aggregation.to_contract() if self.aggregation else None, } + + +def _arrow_table_logical_sha256(table: pa.Table) -> str: + combined = table.combine_chunks() + sink = pa.BufferOutputStream() + with ipc.new_stream(sink, combined.schema) as writer: + writer.write_table(combined) + return hashlib.sha256(sink.getvalue().to_pybytes()).hexdigest() diff --git a/src/quant_data_kit/research_inputs_v2.py b/src/quant_data_kit/research_inputs_v2.py index 819ab98..3c8ba73 100644 --- a/src/quant_data_kit/research_inputs_v2.py +++ b/src/quant_data_kit/research_inputs_v2.py @@ -17,9 +17,14 @@ import pyarrow.parquet as pq from pyarrow import ipc -from quant_data_kit.curated import build_event_bars, load_curated_snapshot +from quant_data_kit.curated import ( + _validate_curated_partition_table, + build_event_bars, + load_curated_snapshot, +) from quant_data_kit.data_lake import ( StoragePolicy, + _json_evidence, _lake_lock, _mkdir_in_lake, _publish_tree_entry, @@ -27,7 +32,6 @@ _stable_staging_directory, _validate_lake_path, load_normalized_snapshot, - read_normalized_events, require_collection_capacity, ) from quant_data_kit.domain_v2 import ( @@ -141,6 +145,143 @@ def _table_logical_sha256(table: pa.Table) -> str: return _hash_bytes(sink.getvalue().to_pybytes()) +def _file_stamp(path: Path) -> tuple[int, int, int, int, int]: + stat = path.stat() + return ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + + +def _read_content_bound_parquet( + path: Path, + expected_content_sha256: str, +) -> tuple[pa.Table, tuple[int, int, int, int, int]]: + """Hash and parse the same in-memory bytes, rejecting concurrent replacement.""" + before = _file_stamp(path) + payload = path.read_bytes() + after_read = _file_stamp(path) + if before != after_read: + raise ValidationError(f"snapshot partition changed while reading: {path}") + if _hash_bytes(payload) != expected_content_sha256: + raise ValidationError(f"snapshot partition bytes differ from its manifest: {path}") + try: + table = pq.ParquetFile(pa.BufferReader(payload)).read() + except (OSError, pa.ArrowException) as exc: + raise ValidationError(f"snapshot partition is not readable Parquet: {path}") from exc + after_parse = _file_stamp(path) + if after_read != after_parse: + raise ValidationError(f"snapshot partition changed while parsing: {path}") + return table, after_parse + + +def _assert_file_stamps( + stamps: Mapping[Path, tuple[int, int, int, int, int]], +) -> None: + for path, expected in stamps.items(): + try: + actual = _file_stamp(path) + except FileNotFoundError as exc: + raise ValidationError(f"snapshot partition disappeared while reading: {path}") from exc + if actual != expected: + raise ValidationError(f"snapshot partition changed during verified read: {path}") + + +def _validate_normalized_partition_table(partition: Any, table: pa.Table) -> None: + validate_arrow_table(partition.schema_id, table) + if table.num_rows != partition.rows: + raise ValidationError("Normalized partition row count differs from its manifest") + previous: tuple[datetime, int, str] | None = None + previous_sequence_by_session: dict[str, int] = {} + for row in table.to_pylist(): + trading_day = row["trading_day"] + trading_day_text = ( + trading_day.isoformat() if isinstance(trading_day, date) else str(trading_day) + ) + if str(row["event_type"]) != partition.event_type: + raise ValidationError("Normalized row event_type differs from its partition") + if str(row["instrument_id"]) != partition.instrument_id: + raise ValidationError("Normalized row instrument differs from its partition") + if trading_day_text != partition.trading_date: + raise ValidationError("Normalized row trading_day differs from its partition") + if str(row["source"]) != partition.provider: + raise ValidationError("Normalized row source differs from its partition provider") + identity = ( + _utc(row["event_time"], "event_time"), + int(row["sequence"]), + str(row["event_id"]), + ) + if previous is not None and identity <= previous: + raise ValidationError("Normalized partition rows are not strictly ordered") + session_id = str(row["session_id"]) + previous_sequence = previous_sequence_by_session.get(session_id) + if previous_sequence is not None and identity[1] <= previous_sequence: + raise ValidationError("Normalized partition sequence does not advance") + previous = identity + previous_sequence_by_session[session_id] = identity[1] + + +def _read_bound_normalized_records( + lake_root: Path, + snapshot: Any, + refs: tuple[EventSchemaRef, ...], +) -> tuple[ + list[tuple[EventSchemaRef, dict[str, Any]]], + dict[Path, tuple[int, int, int, int, int]], +]: + ref_by_schema = {item.schema_id: item for item in refs} + snapshot_dir = _validate_lake_path( + lake_root, + lake_root / "normalized" / "snapshots" / snapshot.snapshot_id, + allow_missing=False, + ) + selected: list[tuple[EventSchemaRef, dict[str, Any]]] = [] + stamps: dict[Path, tuple[int, int, int, int, int]] = {} + for partition in snapshot.partitions: + schema_ref = ref_by_schema.get(partition.schema_id) + if schema_ref is None: + continue + path = _validate_lake_path( + lake_root, snapshot_dir / partition.relative_path, allow_missing=False + ) + table, stamp = _read_content_bound_parquet(path, partition.content_sha256) + _validate_normalized_partition_table(partition, table) + stamps[path] = stamp + selected.extend((schema_ref, _json_evidence(row)) for row in table.to_pylist()) + return selected, stamps + + +def _read_bound_curated_tables( + lake_root: Path, + snapshot: Any, +) -> tuple[ + list[pa.Table], + dict[str, pa.Table], + dict[Path, tuple[int, int, int, int, int]], +]: + snapshot_dir = _validate_lake_path( + lake_root, + lake_root / "curated" / snapshot.dataset / "snapshots" / snapshot.snapshot_id, + allow_missing=False, + ) + tables: list[pa.Table] = [] + tables_by_path: dict[str, pa.Table] = {} + stamps: dict[Path, tuple[int, int, int, int, int]] = {} + for partition in snapshot.partitions: + path = _validate_lake_path( + lake_root, snapshot_dir / partition.relative_path, allow_missing=False + ) + table, stamp = _read_content_bound_parquet(path, partition.content_sha256) + _validate_curated_partition_table(partition, table, snapshot.aggregation) + tables.append(table) + tables_by_path[partition.relative_path] = table + stamps[path] = stamp + return tables, tables_by_path, stamps + + def _record_selection_sha256(schema_id: str, rows: Sequence[Mapping[str, Any]]) -> str: payload = { "algorithm": "puresaber.event-selection-canonical-json@1.0.0", @@ -542,12 +683,12 @@ def _ordered_records( return sorted( records, key=lambda item: ( - str(item[1]["source"]), str(item[1]["instrument_id"]), - str(item[1]["session_id"]), _utc(item[1]["event_time"], "event_time"), int(item[1]["sequence"]), str(item[1]["event_id"]), + str(item[1]["source"]), + str(item[1]["session_id"]), item[0].schema_id, ), ) @@ -555,6 +696,7 @@ def _ordered_records( def _validate_event_order(records: Sequence[tuple[EventSchemaRef, dict[str, Any]]]) -> None: event_ids: set[str] = set() + previous_by_instrument: dict[str, tuple[datetime, int, str]] = {} previous_by_stream: dict[tuple[str, str, str, str], tuple[datetime, int, str]] = {} l2_streams: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) for schema_ref, record in records: @@ -575,6 +717,11 @@ def _validate_event_order(records: Sequence[tuple[EventSchemaRef, dict[str, Any] int(record["sequence"]), event_id, ) + instrument_id = str(record["instrument_id"]) + instrument_previous = previous_by_instrument.get(instrument_id) + if instrument_previous is not None and identity <= instrument_previous: + raise ValidationError(f"instrument events are not strictly ordered: {instrument_id}") + previous_by_instrument[instrument_id] = identity previous = previous_by_stream.get(stream) if previous is not None and identity <= previous: raise ValidationError(f"event stream is not strictly ordered: {stream}") @@ -660,19 +807,11 @@ def load_verified_normalized_events( lake_root = _resolved_lake_root(root, create=False) before = load_normalized_snapshot(lake_root, snapshot_id) context_before = load_market_context_snapshot(lake_root, market_context_snapshot_id) - rows = read_normalized_events(lake_root, before.snapshot_id) - ref_by_type = {_EVENT_TYPE_BY_SCHEMA[item.schema_id]: item for item in refs} - selected: list[tuple[EventSchemaRef, dict[str, Any]]] = [] + selected, partition_stamps = _read_bound_normalized_records(lake_root, before, refs) counts = {item: 0 for item in refs} - for raw in rows: - event_type = str(raw.get("event_type")) - schema_ref = ref_by_type.get(event_type) - if schema_ref is None: - continue - record = dict(raw) + for schema_ref, record in selected: validate_json_record(schema_ref.schema_id, record, schema_ref.schema_version) _validate_context_record(context_before, record, bar=False) - selected.append((schema_ref, record)) counts[schema_ref] += 1 missing = [item.schema_id for item, count in counts.items() if count == 0] if missing: @@ -690,7 +829,8 @@ def load_verified_normalized_events( raise ValidationError("Normalized snapshot changed while building verified input") if context_before.manifest() != context_after.manifest(): raise ValidationError("market context changed while building verified input") - return VerifiedFactorInput( + _assert_file_stamps(partition_stamps) + return VerifiedFactorInput._from_certified_factory( layer="normalized", source_snapshot_id=before.snapshot_id, source_logical_sha256=before.logical_sha256, @@ -819,9 +959,8 @@ def _source_rows_for_evidence( def _verify_event_bars( - root: Path, aggregation: CuratedAggregation, - normalized_snapshot_id: str, + source_rows: Sequence[Mapping[str, Any]], table: pa.Table, context: MarketContextSnapshot, ) -> None: @@ -831,15 +970,19 @@ def _verify_event_bars( EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2), ): raise ValidationError("M8 event bars currently require the frozen Trade schema") - source_rows = read_normalized_events(root, normalized_snapshot_id, event_type="trade") output_sources = set(table.column("source").to_pylist()) if len(output_sources) != 1: raise ValidationError("event Bars must use one deterministic Curated source") output_source = str(next(iter(output_sources))) session_starts = {item.session_id: item.opens_at for item in context.sessions} rebuilt: list[dict[str, Any]] = [] + used_event_ids: set[str] = set() for evidence in aggregation.partition_evidence: selected = _source_rows_for_evidence(source_rows, evidence) + selected_ids = {str(item["event_id"]) for item in selected} + if used_event_ids.intersection(selected_ids): + raise ValidationError("event-bar evidence reuses source events") + used_event_ids.update(selected_ids) rebuilt.extend( build_event_bars( selected, @@ -851,6 +994,9 @@ def _verify_event_bars( require_complete=True, ) ) + all_event_ids = {str(item["event_id"]) for item in source_rows} + if used_event_ids != all_event_ids: + raise ValidationError("event-bar evidence does not cover its complete Trade lineage") expected_table = pa.Table.from_pylist( [_arrow_ready(item, get_arrow_schema(BAR_EVENT_SCHEMA_ID)) for item in rebuilt], schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID), @@ -887,19 +1033,7 @@ def load_verified_curated_bars( or aggregation.session_policy_version != context_before.session_policy_version ): raise ValidationError("Curated aggregation market-context binding changed") - snapshot_dir = _validate_lake_path( - lake_root, - lake_root / "curated" / dataset / "snapshots" / snapshot_id, - allow_missing=False, - ) - tables: list[pa.Table] = [] - for partition in before.partitions: - path = _validate_lake_path( - lake_root, snapshot_dir / partition.relative_path, allow_missing=False - ) - table = pq.ParquetFile(path).read() - validate_arrow_table(BAR_EVENT_SCHEMA_ID, table) - tables.append(table) + tables, tables_by_path, curated_stamps = _read_bound_curated_tables(lake_root, before) if not tables: raise ValidationError("Curated snapshot contains no Bar partitions") combined = pa.concat_tables(tables).combine_chunks() @@ -921,10 +1055,21 @@ def load_verified_curated_bars( partition = partition_by_path[item.relative_path] if item.instrument_id != partition.instrument_id: raise ValidationError("event-bar evidence instrument does not match its partition") + partition_sessions = set( + tables_by_path[item.relative_path].column("session_id").to_pylist() + ) + if partition_sessions != {item.session_id}: + raise ValidationError("event-bar evidence session does not match its partition") + normalized_before = load_normalized_snapshot( + lake_root, before.lineage["normalized_snapshot_id"] + ) + trade_ref = (EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),) + source_pairs, normalized_stamps = _read_bound_normalized_records( + lake_root, normalized_before, trade_ref + ) _verify_event_bars( - lake_root, aggregation, - before.lineage["normalized_snapshot_id"], + [record for _, record in source_pairs], table, context_before, ) @@ -937,7 +1082,12 @@ def load_verified_curated_bars( normalized = load_normalized_snapshot(lake_root, before.lineage["normalized_snapshot_id"]) if normalized.logical_sha256 != before.lineage["normalized_logical_sha256"]: raise ValidationError("Curated Normalized lineage hash changed") - return VerifiedFactorInput( + _assert_file_stamps(curated_stamps) + if aggregation.kind == "event_bar": + if normalized_before != normalized: + raise ValidationError("Normalized lineage changed while verifying event Bars") + _assert_file_stamps(normalized_stamps) + return VerifiedFactorInput._from_certified_factory( layer="curated", source_snapshot_id=before.snapshot_id, source_logical_sha256=before.logical_sha256, diff --git a/tests/test_m8_research_contracts.py b/tests/test_m8_research_contracts.py index 5486daf..dae64f9 100644 --- a/tests/test_m8_research_contracts.py +++ b/tests/test_m8_research_contracts.py @@ -1,10 +1,12 @@ from __future__ import annotations from dataclasses import replace +from datetime import date, datetime, timezone import pyarrow as pa import pytest +import quant_data_kit.research_contracts_v2 as contracts from quant_data_kit.exceptions import ValidationError from quant_data_kit.fixed_point import FixedPoint from quant_data_kit.research_contracts_v2 import ( @@ -18,9 +20,11 @@ VerifiedFactorInput, ) from quant_data_kit.schemas_v2 import ( + BAR_EVENT_SCHEMA_ID, BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2, TRADE_EVENT_SCHEMA_ID, + get_arrow_schema, ) HASH = "0" * 64 @@ -159,14 +163,37 @@ def test_aggregation_kind_conditions_and_order_are_strict() -> None: with pytest.raises(ValidationError, match="evidence"): replace(event, partition_evidence=()) duplicate = evidence(first=3, last=4) - with pytest.raises(ValidationError, match="globally unique"): + with pytest.raises(ValidationError, match="partition paths"): replace(event, partition_evidence=(evidence(), duplicate)) - second = evidence(first=3, last=4, digest=OTHER_HASH) + second = evidence( + first=3, + last=4, + digest=OTHER_HASH, + path="date=2026-01-05/instrument=IF/session=pm/data.parquet", + ) with pytest.raises(ValidationError, match="sorted"): replace(event, partition_evidence=(second, evidence())) - overlap = evidence(first=2, last=3, digest=OTHER_HASH) + overlap = evidence( + first=2, + last=3, + digest=OTHER_HASH, + path="date=2026-01-05/instrument=IF/session=pm/data.parquet", + ) with pytest.raises(ValidationError, match="overlap"): replace(event, partition_evidence=(evidence(), overlap)) + excessive_count = replace( + second, + first_sequence=3, + last_sequence=3, + event_count=2, + first_event_id="e3", + last_event_id="e3", + ) + with pytest.raises(ValidationError, match="event_count"): + replace(event, partition_evidence=(evidence(), excessive_count)) + bad_boundary = replace(second, first_sequence=3, last_sequence=3, event_count=1) + with pytest.raises(ValidationError, match="boundary"): + replace(event, partition_evidence=(evidence(), bad_boundary)) def test_aggregation_parser_rejects_noncanonical_and_open_payloads() -> None: @@ -194,13 +221,19 @@ def test_aggregation_parser_rejects_noncanonical_and_open_payloads() -> None: def valid_verified_input(**changes) -> VerifiedFactorInput: + table = pa.table( + { + "event_schema_id": [TRADE_EVENT_SCHEMA_ID], + "value": [1], + } + ) values = { "layer": "normalized", "source_snapshot_id": SNAPSHOT, "source_logical_sha256": HASH, - "selection_logical_sha256": OTHER_HASH, + "selection_logical_sha256": contracts._arrow_table_logical_sha256(table), "event_schemas": (TRADE_SCHEMA,), - "table": pa.table({"value": [1]}), + "table": table, "calendar_id": "calendar-v1", "session_policy_version": "sessions-v1", "market_context_snapshot_id": SNAPSHOT, @@ -208,7 +241,39 @@ def valid_verified_input(**changes) -> VerifiedFactorInput: "lineage": (LineageRef("market", SNAPSHOT, HASH),), } values.update(changes) - return VerifiedFactorInput(**values) + if "table" in changes and "selection_logical_sha256" not in changes: + values["selection_logical_sha256"] = contracts._arrow_table_logical_sha256(values["table"]) + return VerifiedFactorInput._from_certified_factory(**values) + + +def bar_table() -> pa.Table: + timestamp = datetime(2026, 1, 5, 1, 31, tzinfo=timezone.utc) + fixed = {"units": 1, "scale": 0} + return pa.Table.from_pylist( + [ + { + "event_type": "bar", + "event_id": "bar-1", + "instrument_id": "IF", + "event_time": timestamp, + "received_at": timestamp, + "available_at": timestamp, + "source": "fixture", + "trading_day": date(2026, 1, 5), + "session_id": "day", + "sequence": 1, + "bar_start": datetime(2026, 1, 5, 1, 30, tzinfo=timezone.utc), + "bar_end": timestamp, + "open_price": fixed, + "high_price": fixed, + "low_price": fixed, + "close_price": fixed, + "volume": fixed, + "is_complete": True, + } + ], + schema=get_arrow_schema(BAR_EVENT_SCHEMA_ID), + ) def test_verified_input_is_closed_nonempty_and_layer_safe() -> None: @@ -227,8 +292,44 @@ def test_verified_input_is_closed_nonempty_and_layer_safe() -> None: for changes in cases: with pytest.raises(ValidationError): valid_verified_input(**changes) - curated = valid_verified_input(layer="curated", aggregation=fixed_aggregation()) + curated_table = bar_table() + curated = valid_verified_input( + layer="curated", + aggregation=fixed_aggregation(), + event_schemas=(EventSchemaRef(BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + table=curated_table, + ) assert curated.to_contract()["aggregation"]["kind"] == "fixed_time_bar" + with pytest.raises(ValidationError, match="certified factory"): + VerifiedFactorInput( + layer="normalized", + source_snapshot_id=SNAPSHOT, + source_logical_sha256=HASH, + selection_logical_sha256=valid.selection_logical_sha256, + event_schemas=(TRADE_SCHEMA,), + table=valid.table, + calendar_id="calendar-v1", + session_policy_version="sessions-v1", + market_context_snapshot_id=SNAPSHOT, + market_context_logical_sha256=HASH, + lineage=(LineageRef("market", SNAPSHOT, HASH),), + ) + with pytest.raises(ValidationError, match="selection hash"): + valid_verified_input(selection_logical_sha256=OTHER_HASH) + with pytest.raises(ValidationError, match="context differs"): + valid_verified_input( + layer="curated", + aggregation=fixed_aggregation(), + event_schemas=(EventSchemaRef(BAR_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + table=curated_table, + calendar_id="other-calendar", + ) + with pytest.raises(ValidationError, match="frozen Bar schema"): + valid_verified_input( + layer="curated", + aggregation=fixed_aggregation(), + table=curated_table, + ) def test_schema_refs_must_be_sorted_when_multiple() -> None: diff --git a/tests/test_m8_research_inputs.py b/tests/test_m8_research_inputs.py index 8922109..eb8b2c9 100644 --- a/tests/test_m8_research_inputs.py +++ b/tests/test_m8_research_inputs.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json from copy import copy from dataclasses import replace @@ -7,6 +8,7 @@ from pathlib import Path import pyarrow as pa +import pyarrow.parquet as pq import pytest import quant_data_kit.curated as curated_module @@ -664,24 +666,19 @@ def bars(rows: list[dict]) -> pa.Table: broken = copy(event_input.aggregation) object.__setattr__(broken, "partition_evidence", None) with pytest.raises(ValidationError, match="metadata is incomplete"): - research_inputs._verify_event_bars( - tmp_path, broken, source.snapshot_id, event_input.table, context - ) + research_inputs._verify_event_bars(broken, source_rows, event_input.table, context) wrong_schema = replace( event_input.aggregation, source_event_schemas=(EventSchemaRef(BOOK_DELTA_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), ) with pytest.raises(ValidationError, match="Trade schema"): - research_inputs._verify_event_bars( - tmp_path, wrong_schema, source.snapshot_id, event_input.table, context - ) + research_inputs._verify_event_bars(wrong_schema, source_rows, event_input.table, context) multiple_sources = event_input.table.to_pylist() multiple_sources[1] = dict(multiple_sources[1], source="other") with pytest.raises(ValidationError, match="one deterministic"): research_inputs._verify_event_bars( - tmp_path, event_input.aggregation, - source.snapshot_id, + source_rows, bars(multiple_sources), context, ) @@ -690,7 +687,7 @@ def bars(rows: list[dict]) -> pa.Table: changed[0]["close_price"] = dict(changed[0]["close_price"], units=40002) with pytest.raises(ValidationError, match="do not recompute"): research_inputs._verify_event_bars( - tmp_path, event_input.aggregation, source.snapshot_id, bars(changed), context + event_input.aggregation, source_rows, bars(changed), context ) @@ -890,6 +887,295 @@ def test_curated_aggregation_builders_cover_all_kinds_and_reject_bad_inputs() -> ) +def two_session_context( + root: Path, + *, + first_id: str = "z-morning", + second_id: str = "a-afternoon", +): + base = market_context(root) + first = replace( + base.sessions[0], + session_id=first_id, + opens_at=datetime(2026, 1, 5, 1, 30, tzinfo=UTC), + closes_at=datetime(2026, 1, 5, 1, 40, tzinfo=UTC), + ) + second = replace( + base.sessions[0], + session_id=second_id, + opens_at=datetime(2026, 1, 5, 1, 40, tzinfo=UTC), + closes_at=datetime(2026, 1, 5, 2, 0, tzinfo=UTC), + ) + return create_market_context_snapshot( + root, + calendar_id=base.calendar_id, + session_policy_version="cffex-two-session-v1", + instruments=base.instruments, + sessions=[first, second], + policy=TEST_POLICY, + ) + + +def two_session_trades(first_id: str, second_id: str) -> list[dict]: + rows = [ + trade("t1", "2026-01-05T01:30:01Z", 1, 40001), + trade("t2", "2026-01-05T01:30:20Z", 2, 40003), + trade("t3", "2026-01-05T01:40:01Z", 1, 40002), + trade("t4", "2026-01-05T01:40:20Z", 2, 40004), + ] + for row in rows[:2]: + row["session_id"] = first_id + for row in rows[2:]: + row["session_id"] = second_id + return rows + + +def test_multi_session_event_bars_have_unique_certified_partitions(tmp_path: Path) -> None: + context = two_session_context(tmp_path) + source = normalized(tmp_path, two_session_trades("z-morning", "a-afternoon")) + snapshot = curate_trade_event_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="multi-session-event-bars", + revision_id="r1", + recipe_version="event-v1", + basis="trade_count", + threshold=FixedPoint(2, 0), + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + verified = load_verified_curated_bars( + tmp_path, "multi-session-event-bars", snapshot.snapshot_id + ) + evidence = verified.aggregation.partition_evidence if verified.aggregation else None + assert evidence is not None + paths = [item.relative_path for item in evidence] + assert len(paths) == len(set(paths)) == 2 + assert all("/session=" in item for item in paths) + assert {item.session_id for item in evidence} == {"z-morning", "a-afternoon"} + + +def test_normalized_factory_orders_by_event_time_not_session_text(tmp_path: Path) -> None: + context = two_session_context(tmp_path) + source = normalized(tmp_path, two_session_trades("z-morning", "a-afternoon")) + verified = load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + assert verified.table.column("session_id").to_pylist() == [ + "z-morning", + "z-morning", + "a-afternoon", + "a-afternoon", + ] + assert verified.table.column("event_id").to_pylist() == ["t1", "t2", "t3", "t4"] + + +def test_bound_reader_rejects_actual_consumed_bytes_that_do_not_match_manifest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + context = market_context(tmp_path) + target = ( + tmp_path + / "normalized" + / "snapshots" + / source.snapshot_id + / source.partitions[0].relative_path + ) + original_read_bytes = Path.read_bytes + + def changed_bytes(path: Path) -> bytes: + payload = original_read_bytes(path) + if path == target: + return payload[:-1] + bytes([payload[-1] ^ 1]) + return payload + + monkeypatch.setattr(Path, "read_bytes", changed_bytes) + with pytest.raises(ValidationError, match="bytes differ from its manifest"): + load_verified_normalized_events( + tmp_path, + source.snapshot_id, + [EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2)], + context.snapshot_id, + ) + + +def test_partition_row_binding_and_source_order_fail_closed(tmp_path: Path) -> None: + source = normalized( + tmp_path, + [ + trade("t1", "2026-01-05T01:30:01Z", 1), + trade("t2", "2026-01-05T01:30:02Z", 2), + ], + ) + normalized_path = ( + tmp_path + / "normalized" + / "snapshots" + / source.snapshot_id + / source.partitions[0].relative_path + ) + normalized_table = pq.read_table(normalized_path) + reversed_table = normalized_table.take(pa.array([1, 0])) + with pytest.raises(ValidationError, match="strictly ordered"): + research_inputs._validate_normalized_partition_table(source.partitions[0], reversed_table) + wrong_normalized_rows = normalized_table.to_pylist() + wrong_normalized_rows[0] = dict(wrong_normalized_rows[0], instrument_id="OTHER") + wrong_normalized = pa.Table.from_pylist(wrong_normalized_rows, schema=normalized_table.schema) + with pytest.raises(ValidationError, match="instrument differs"): + research_inputs._validate_normalized_partition_table(source.partitions[0], wrong_normalized) + + context = market_context(tmp_path) + curated = curate_trade_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="partition-binding", + revision_id="r1", + recipe_version="fixed-v1", + interval=timedelta(minutes=1), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + curated_path = ( + tmp_path + / "curated" + / "partition-binding" + / "snapshots" + / curated.snapshot_id + / curated.partitions[0].relative_path + ) + curated_table = pq.read_table(curated_path) + wrong_curated_rows = curated_table.to_pylist() + wrong_curated_rows[0] = dict(wrong_curated_rows[0], trading_day=date(2026, 1, 6)) + wrong_curated = pa.Table.from_pylist(wrong_curated_rows, schema=curated_table.schema) + with pytest.raises(ValidationError, match="trading_day"): + curated_module._validate_curated_partition_table( + curated.partitions[0], wrong_curated, curated.aggregation + ) + + +def test_partition_binding_helpers_cover_all_metadata_and_order_guards(tmp_path: Path) -> None: + source = normalized( + tmp_path, + [ + trade("t1", "2026-01-05T01:30:01Z", 1), + trade("t2", "2026-01-05T01:30:02Z", 2), + ], + ) + partition = source.partitions[0] + path = tmp_path / "normalized" / "snapshots" / source.snapshot_id / partition.relative_path + table = pq.read_table(path) + with pytest.raises(ValidationError, match="row count"): + research_inputs._validate_normalized_partition_table( + replace(partition, rows=partition.rows + 1), table + ) + metadata_cases = [ + (replace(partition, event_type="quote"), "event_type"), + (replace(partition, trading_date="2026-01-06"), "trading_day"), + (replace(partition, provider="other"), "source"), + ] + for changed_partition, message in metadata_cases: + with pytest.raises(ValidationError, match=message): + research_inputs._validate_normalized_partition_table(changed_partition, table) + duplicate_sequence_rows = table.to_pylist() + duplicate_sequence_rows[1] = dict(duplicate_sequence_rows[1], sequence=1) + duplicate_sequence = pa.Table.from_pylist(duplicate_sequence_rows, schema=table.schema) + with pytest.raises(ValidationError, match="sequence does not advance"): + research_inputs._validate_normalized_partition_table(partition, duplicate_sequence) + + context = market_context(tmp_path) + event = curate_trade_event_bars_from_snapshot( + tmp_path, + normalized_snapshot_id=source.snapshot_id, + dataset="partition-helper-event", + revision_id="r1", + recipe_version="event-v1", + basis="trade_count", + threshold=FixedPoint(2, 0), + market_context_snapshot_id=context.snapshot_id, + policy=TEST_POLICY, + ) + event_partition = event.partitions[0] + event_path = ( + tmp_path + / "curated" + / "partition-helper-event" + / "snapshots" + / event.snapshot_id + / event_partition.relative_path + ) + event_table = pq.read_table(event_path) + with pytest.raises(ValidationError, match="row count"): + curated_module._validate_curated_partition_table( + replace(event_partition, rows=event_partition.rows + 1), + event_table, + event.aggregation, + ) + wrong_instrument_rows = event_table.to_pylist() + wrong_instrument_rows[0] = dict(wrong_instrument_rows[0], instrument_id="OTHER") + wrong_instrument = pa.Table.from_pylist(wrong_instrument_rows, schema=event_table.schema) + with pytest.raises(ValidationError, match="instrument"): + curated_module._validate_curated_partition_table( + event_partition, wrong_instrument, event.aggregation + ) + multiple_session_rows = event_table.to_pylist() + multiple_session_rows.append( + dict( + multiple_session_rows[0], + event_id="other-session", + event_time=datetime(2026, 1, 5, 1, 31, tzinfo=UTC), + received_at=datetime(2026, 1, 5, 1, 31, tzinfo=UTC), + available_at=datetime(2026, 1, 5, 1, 31, tzinfo=UTC), + bar_end=datetime(2026, 1, 5, 1, 31, tzinfo=UTC), + sequence=event_table.num_rows + 1, + session_id="other-session", + ) + ) + multiple_session = pa.Table.from_pylist(multiple_session_rows, schema=event_table.schema) + expanded_partition = replace(event_partition, rows=multiple_session.num_rows) + with pytest.raises(ValidationError, match="exactly one session"): + curated_module._validate_curated_partition_table( + expanded_partition, multiple_session, event.aggregation + ) + + +def test_content_bound_reader_and_final_stamp_failure_branches( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = normalized(tmp_path, [trade("t1", "2026-01-05T01:30:01Z", 1)]) + partition = source.partitions[0] + path = tmp_path / "normalized" / "snapshots" / source.snapshot_id / partition.relative_path + original_stamp = research_inputs._file_stamp + calls = 0 + + def changed_during_read(target: Path): + nonlocal calls + calls += 1 + stamp = original_stamp(target) + return (*stamp[:-1], stamp[-1] + 1) if calls == 2 else stamp + + monkeypatch.setattr(research_inputs, "_file_stamp", changed_during_read) + with pytest.raises(ValidationError, match="changed while reading"): + research_inputs._read_content_bound_parquet(path, partition.content_sha256) + monkeypatch.setattr(research_inputs, "_file_stamp", original_stamp) + + invalid = tmp_path / "invalid.parquet" + invalid.write_bytes(b"not parquet") + with pytest.raises(ValidationError, match="not readable Parquet"): + research_inputs._read_content_bound_parquet( + invalid, hashlib.sha256(b"not parquet").hexdigest() + ) + with pytest.raises(ValidationError, match="changed during verified read"): + research_inputs._assert_file_stamps({path: (0, 0, 0, 0, 0)}) + missing = tmp_path / "missing.parquet" + with pytest.raises(ValidationError, match="disappeared"): + research_inputs._assert_file_stamps({missing: (0, 0, 0, 0, 0)}) + + def test_trading_day_rollup_uses_all_authoritative_sessions(tmp_path: Path) -> None: morning_id = "CFFEX-IF-2026-01-05-AM" afternoon_id = "CFFEX-IF-2026-01-05-PM" From c1277a2c6b1ba144645bf107b48bb632d0c4fcd4 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 11:27:08 +0800 Subject: [PATCH 3/4] fix: seal verified input provenance --- src/quant_data_kit/curated.py | 58 +++++++++++---- src/quant_data_kit/research_contracts_v2.py | 20 ++++-- tests/test_m8_research_contracts.py | 11 ++- tests/test_m8_research_inputs.py | 78 +++++++++++++++++++++ 4 files changed, 147 insertions(+), 20 deletions(-) diff --git a/src/quant_data_kit/curated.py b/src/quant_data_kit/curated.py index f3bf742..3980c71 100644 --- a/src/quant_data_kit/curated.py +++ b/src/quant_data_kit/curated.py @@ -654,6 +654,7 @@ def _write_curated_bars( normalized_snapshot_id: str, policy: StoragePolicy, aggregation: CuratedAggregation | None = None, + event_partition_scopes: Mapping[str, tuple[str, str]] | None = None, ) -> CuratedSnapshot: """Persist bars only after loading their exact immutable Normalized lineage.""" lake_root = _resolved_lake_root(root, create=False) @@ -672,12 +673,27 @@ def _write_curated_bars( if not records: raise ValidationError("Cannot write an empty Curated snapshot") event_partitioned = aggregation is not None and aggregation.kind == "event_bar" + if event_partitioned and event_partition_scopes is None: + raise ValidationError("event Bars require source/session partition scopes") + if not event_partitioned and event_partition_scopes is not None: + raise ValidationError("source/session partition scopes are only valid for event Bars") + scopes = event_partition_scopes groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) for record in records: validate_json_record(BAR_EVENT_SCHEMA_ID, record) key = (str(record["trading_day"]), str(record["instrument_id"])) if event_partitioned: - key += (str(record["session_id"]),) + assert scopes is not None + scope = scopes.get(str(record["event_id"])) + if ( + scope is None + or len(scope) != 2 + or not all(isinstance(item, str) and item for item in scope) + ): + raise ValidationError("event Bar lacks a valid source/session partition scope") + if scope[1] != str(record["session_id"]): + raise ValidationError("event Bar partition scope session differs from its row") + key += scope groups[key].append(record) estimated_bytes = sum(len(_canonical(_json_value(item))) for item in records) @@ -709,7 +725,9 @@ def _write_curated_bars( validate_arrow_table(BAR_EVENT_SCHEMA_ID, table) partition_root = f"date={trading_date}/instrument={quote(instrument_id, safe='-._')}" if event_partitioned: - partition_root += f"/session={quote(key[2], safe='-._')}" + partition_root += ( + f"/source={quote(key[2], safe='-._')}/session={quote(key[3], safe='-._')}" + ) relative = Path(f"{partition_root}/data.parquet") target = stage / relative _mkdir_in_lake(lake_root, target.parent) @@ -930,19 +948,7 @@ def curate_trade_event_bars_from_snapshot( ] if not trades: raise ValidationError("Normalized snapshot contains no trades to curate") - upstream_sources = {str(item["source"]) for item in trades} - if len(upstream_sources) != 1: - raise ValidationError("certified event Bars require one upstream source per snapshot") session_starts = {item.session_id: item.opens_at for item in context.sessions} - bars = build_event_bars( - trades, - basis=basis, - threshold=threshold, - session_starts=session_starts, - source=source, - recipe_version=recipe_version, - require_complete=True, - ) grouped: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) for record in trades: grouped[ @@ -953,6 +959,8 @@ def curate_trade_event_bars_from_snapshot( str(record["session_id"]), ) ].append(record) + bars: list[dict[str, Any]] = [] + event_partition_scopes: dict[str, tuple[str, str]] = {} evidence: list[EventBarPartitionEvidence] = [] for (upstream_source, instrument_id, trading_day, session_id), rows in sorted(grouped.items()): ordered = sorted( @@ -963,8 +971,24 @@ def curate_trade_event_bars_from_snapshot( str(item["event_id"]), ), ) + stream_bars = build_event_bars( + ordered, + basis=basis, + threshold=threshold, + session_starts=session_starts, + source=source, + recipe_version=recipe_version, + require_complete=True, + ) + bars.extend(stream_bars) + for bar in stream_bars: + event_id = str(bar["event_id"]) + if event_id in event_partition_scopes: + raise ValidationError("event Bar identity collides across source streams") + event_partition_scopes[event_id] = (upstream_source, session_id) relative_path = Path( f"date={trading_day}/instrument={quote(instrument_id, safe='-._')}/" + f"source={quote(upstream_source, safe='-._')}/" f"session={quote(session_id, safe='-._')}/data.parquet" ).as_posix() evidence.append( @@ -1013,6 +1037,7 @@ def curate_trade_event_bars_from_snapshot( normalized_snapshot_id=normalized.snapshot_id, policy=policy or StoragePolicy(), aggregation=aggregation, + event_partition_scopes=event_partition_scopes, ) @@ -1142,7 +1167,10 @@ def _load_curated_snapshot( if evidence is None or evidence.instrument_id != partition.instrument_id: raise ValidationError("Curated partition path metadata mismatch") expected_event_session = evidence.session_id - partition_root += f"/session={quote(expected_event_session, safe='-._')}" + partition_root += ( + f"/source={quote(evidence.source, safe='-._')}/" + f"session={quote(expected_event_session, safe='-._')}" + ) expected_relative = Path(f"{partition_root}/data.parquet").as_posix() if partition.relative_path != expected_relative: raise ValidationError("Curated partition path metadata mismatch") diff --git a/src/quant_data_kit/research_contracts_v2.py b/src/quant_data_kit/research_contracts_v2.py index 8a5bc66..1b9ce1e 100644 --- a/src/quant_data_kit/research_contracts_v2.py +++ b/src/quant_data_kit/research_contracts_v2.py @@ -5,7 +5,7 @@ import hashlib import re from collections.abc import Mapping -from dataclasses import dataclass, field +from dataclasses import InitVar, dataclass, field from typing import Any, Literal import pyarrow as pa @@ -396,10 +396,10 @@ class VerifiedFactorInput: lineage: tuple[LineageRef, ...] = () aggregation: CuratedAggregation | None = None schema_id: str = VERIFIED_FACTOR_INPUT_SCHEMA_ID - _factory_token: object = field(default=None, repr=False, compare=False) + _factory_token: InitVar[object] = None - def __post_init__(self) -> None: - if self._factory_token is not _VERIFIED_INPUT_FACTORY_TOKEN: + def __post_init__(self, _factory_token: object) -> None: + if _factory_token is not _VERIFIED_INPUT_FACTORY_TOKEN: raise ValidationError("VerifiedFactorInput can only be created by a certified factory") if self.schema_id != VERIFIED_FACTOR_INPUT_SCHEMA_ID: raise ValidationError("unsupported VerifiedFactorInput schema") @@ -425,6 +425,18 @@ def __post_init__(self) -> None: lineage_keys = [(item.role, item.snapshot_id) for item in lineage] if len(lineage_keys) != len(set(lineage_keys)): raise ValidationError("lineage roles and snapshots must be unique") + source_lineage = [item for item in lineage if item.role == "market"] + if len(source_lineage) != 1 or ( + source_lineage[0].snapshot_id, + source_lineage[0].logical_sha256, + ) != (self.source_snapshot_id, self.source_logical_sha256): + raise ValidationError("source snapshot differs from its market lineage") + context_lineage = [item for item in lineage if item.role == "market_context"] + if len(context_lineage) != 1 or ( + context_lineage[0].snapshot_id, + context_lineage[0].logical_sha256, + ) != (self.market_context_snapshot_id, self.market_context_logical_sha256): + raise ValidationError("market context differs from its lineage") if self.layer == "curated": if self.aggregation is None: raise ValidationError("Curated verified input requires aggregation metadata") diff --git a/tests/test_m8_research_contracts.py b/tests/test_m8_research_contracts.py index dae64f9..c840b62 100644 --- a/tests/test_m8_research_contracts.py +++ b/tests/test_m8_research_contracts.py @@ -238,7 +238,10 @@ def valid_verified_input(**changes) -> VerifiedFactorInput: "session_policy_version": "sessions-v1", "market_context_snapshot_id": SNAPSHOT, "market_context_logical_sha256": HASH, - "lineage": (LineageRef("market", SNAPSHOT, HASH),), + "lineage": ( + LineageRef("market", SNAPSHOT, HASH), + LineageRef("market_context", SNAPSHOT, HASH), + ), } values.update(changes) if "table" in changes and "selection_logical_sha256" not in changes: @@ -314,6 +317,12 @@ def test_verified_input_is_closed_nonempty_and_layer_safe() -> None: market_context_logical_sha256=HASH, lineage=(LineageRef("market", SNAPSHOT, HASH),), ) + with pytest.raises(ValidationError, match="certified factory"): + replace(valid, source_snapshot_id=f"sha256-{OTHER_HASH}") + with pytest.raises(ValidationError, match="market lineage"): + valid_verified_input(source_snapshot_id=f"sha256-{OTHER_HASH}") + with pytest.raises(ValidationError, match="context differs from its lineage"): + valid_verified_input(market_context_logical_sha256=OTHER_HASH) with pytest.raises(ValidationError, match="selection hash"): valid_verified_input(selection_logical_sha256=OTHER_HASH) with pytest.raises(ValidationError, match="context differs"): diff --git a/tests/test_m8_research_inputs.py b/tests/test_m8_research_inputs.py index eb8b2c9..0833a74 100644 --- a/tests/test_m8_research_inputs.py +++ b/tests/test_m8_research_inputs.py @@ -36,6 +36,8 @@ from quant_data_kit.exceptions import ValidationError from quant_data_kit.fixed_point import FixedPoint from quant_data_kit.research_contracts_v2 import ( + CuratedAggregation, + EventBarPartitionEvidence, EventSchemaRef, ) from quant_data_kit.research_inputs_v2 import ( @@ -951,6 +953,7 @@ def test_multi_session_event_bars_have_unique_certified_partitions(tmp_path: Pat assert evidence is not None paths = [item.relative_path for item in evidence] assert len(paths) == len(set(paths)) == 2 + assert all("/source=cn-fixture/" in item for item in paths) assert all("/session=" in item for item in paths) assert {item.session_id for item in evidence} == {"z-morning", "a-afternoon"} @@ -973,6 +976,81 @@ def test_normalized_factory_orders_by_event_time_not_session_text(tmp_path: Path assert verified.table.column("event_id").to_pylist() == ["t1", "t2", "t3", "t4"] +def test_event_bar_writer_partitions_independent_sources_without_rejecting_them( + tmp_path: Path, +) -> None: + primary = [ + trade("p1", "2026-01-05T01:30:01Z", 1), + trade("p2", "2026-01-05T01:30:02Z", 2), + ] + source = normalized(tmp_path, primary) + context = market_context(tmp_path) + secondary = [ + dict(trade("s1", "2026-01-05T01:30:03Z", 1), source="other"), + dict(trade("s2", "2026-01-05T01:30:04Z", 2), source="other"), + ] + streams = [("cn-fixture", primary), ("other", secondary)] + bars: list[dict] = [] + scopes: dict[str, tuple[str, str]] = {} + evidence: list[EventBarPartitionEvidence] = [] + for upstream_source, rows in streams: + stream_bars = build_event_bars( + rows, + basis="trade_count", + threshold=FixedPoint(2, 0), + session_starts={SESSION_ID: datetime(2026, 1, 5, 1, 30, tzinfo=UTC)}, + source="curated", + recipe_version="multi-source-v1", + ) + bars.extend(stream_bars) + scopes.update({str(bar["event_id"]): (upstream_source, SESSION_ID) for bar in stream_bars}) + relative_path = ( + "date=2026-01-05/instrument=IF-CONT/" + f"source={upstream_source}/session={SESSION_ID}/data.parquet" + ) + evidence.append( + EventBarPartitionEvidence( + relative_path=relative_path, + source=upstream_source, + instrument_id="IF-CONT", + session_id=SESSION_ID, + first_sequence=1, + last_sequence=2, + first_event_id=str(rows[0]["event_id"]), + last_event_id=str(rows[-1]["event_id"]), + event_count=2, + source_selection_sha256=curated_module._source_selection_sha256(rows), + ) + ) + aggregation = CuratedAggregation( + calendar_id=context.calendar_id, + session_policy_version=context.session_policy_version, + kind="event_bar", + recipe_version="multi-source-v1", + event_bar_basis="trade_count", + event_bar_threshold=FixedPoint(2, 0), + market_context_snapshot_id=context.snapshot_id, + market_context_logical_sha256=context.logical_sha256, + source_event_schemas=(EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), + partition_evidence=tuple(evidence), + ) + snapshot = curated_module._write_curated_bars( + tmp_path, + bars, + dataset="multi-source-writer", + revision_id="r1", + recipe_version="multi-source-v1", + normalized_snapshot_id=source.snapshot_id, + policy=TEST_POLICY, + aggregation=aggregation, + event_partition_scopes=scopes, + ) + assert len(snapshot.partitions) == 2 + assert {item.relative_path for item in snapshot.partitions} == { + item.relative_path for item in evidence + } + + def test_bound_reader_rejects_actual_consumed_bytes_that_do_not_match_manifest( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From fa2adfbb28457bf60b6e919ab6b478b3d74b13f8 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Mon, 31 Aug 2026 11:33:32 +0800 Subject: [PATCH 4/4] test: clarify provider-bound event lineage --- README.md | 1 + tests/test_m8_research_inputs.py | 56 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bdc6a35..358e463 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ with `legacy-curated-not-m8-certified` at the certified factory. - Raw persists exact provider bytes through lake-local staging and atomic rename. Its integrity anchor binds source, request, UTC collection time, object/key path identity, SHA-256 and the30-day retention policy. A crash-released process lock and immutable key claim make concurrent writes, crash recovery and cleanup serialize on the same idempotency key. Every write, read and cleanup rejects path escapes and Windows reparse points below the lake root. - Normalized requires resolvable, hash-verified Raw references and writes only frozen`standard/v2`Arrow schemas under`provider/venue/event_type/date/instrument`partitions. Capture epochs persist`PREPARED`before snapshot publication and finish as`COMMITTED`or`ABORTED`; startup uses the frozen stream configuration as an independent identity anchor, enforces closed terminal JSON fields and strict types, recomputes partition rows, logical hashes, the available-time maximum and the final L2 state from the immutable journal, and rejects any receipt bound to a different snapshot before network startup. A sharded persistent claim index binds every lake-wide`event_id`to its Arrow-normalized logical event hash. Same-ID/same-content reuse is idempotent; conflicting content, bad sequences and L2 reconstruction failures cannot enter research snapshots. - Certified Curated producers are`curate_trade_bars_from_snapshot`、`curate_session_bars_from_snapshot`and`curate_trade_event_bars_from_snapshot`: they read trades from one explicit verified Normalized snapshot, construct authoritative fixed/session/event Bars and bind exact lineage plus an immutable market-context snapshot. One`dataset+revision_id`maps to one snapshot; corrected data requires a new revision and never overwrites history. +- A Normalized snapshot is provider-bound. Binance and OKX certification therefore uses separate immutable snapshots and separate verified inputs; a Curated writer can partition evidence by`source/session`, but the certified loader rejects any source that is not present in the snapshot lineage. - Normalized and Curated snapshot identities bind Arrow-canonical logical rows and physical Parquet hashes. DuckDB verifies the fixed snapshot, copies Arrow data into in-memory tables, then disables external access; user SQL cannot call file readers or resolve`latest`/`main`. - Collection stops with a visible`COLLECTION_STOPPED`error if hot data would exceed150GB or free space would fall below`max(volume*20%,100GB)`. - Raw cleanup requires all of: the30-day window elapsed, explicit confirmation, an accessible real local archive object, archive hash equality and a successful restore-hash exercise. Cleanup publishes an immutable audit tombstone and resumes an explicit`deleting`state after interruption; local unlink failures remain visible to callers. Remote archives have no M2 verifier and therefore stop cleanup. No background or silent deletion path exists. diff --git a/tests/test_m8_research_inputs.py b/tests/test_m8_research_inputs.py index 0833a74..e2badf9 100644 --- a/tests/test_m8_research_inputs.py +++ b/tests/test_m8_research_inputs.py @@ -976,7 +976,7 @@ def test_normalized_factory_orders_by_event_time_not_session_text(tmp_path: Path assert verified.table.column("event_id").to_pylist() == ["t1", "t2", "t3", "t4"] -def test_event_bar_writer_partitions_independent_sources_without_rejecting_them( +def test_event_bar_writer_partitions_sources_but_certification_enforces_snapshot_lineage( tmp_path: Path, ) -> None: primary = [ @@ -1034,6 +1034,58 @@ def test_event_bar_writer_partitions_independent_sources_without_rejecting_them( source_event_schemas=(EventSchemaRef(TRADE_EVENT_SCHEMA_ID, SCHEMA_VERSION_V2),), partition_evidence=tuple(evidence), ) + with pytest.raises(ValidationError, match="require source/session partition scopes"): + curated_module._write_curated_bars( + tmp_path, + bars, + dataset="missing-event-scopes", + revision_id="r1", + recipe_version="multi-source-v1", + normalized_snapshot_id=source.snapshot_id, + policy=TEST_POLICY, + aggregation=aggregation, + ) + with pytest.raises(ValidationError, match="only valid for event Bars"): + curated_module._write_curated_bars( + tmp_path, + bars, + dataset="unexpected-event-scopes", + revision_id="r1", + recipe_version="multi-source-v1", + normalized_snapshot_id=source.snapshot_id, + policy=TEST_POLICY, + event_partition_scopes=scopes, + ) + incomplete_scopes = dict(scopes) + incomplete_scopes.pop(next(iter(incomplete_scopes))) + with pytest.raises(ValidationError, match="lacks a valid"): + curated_module._write_curated_bars( + tmp_path, + bars, + dataset="incomplete-event-scopes", + revision_id="r1", + recipe_version="multi-source-v1", + normalized_snapshot_id=source.snapshot_id, + policy=TEST_POLICY, + aggregation=aggregation, + event_partition_scopes=incomplete_scopes, + ) + wrong_session_scopes = { + event_id: (upstream_source, "wrong-session") + for event_id, (upstream_source, _session_id) in scopes.items() + } + with pytest.raises(ValidationError, match="session differs"): + curated_module._write_curated_bars( + tmp_path, + bars, + dataset="wrong-session-scopes", + revision_id="r1", + recipe_version="multi-source-v1", + normalized_snapshot_id=source.snapshot_id, + policy=TEST_POLICY, + aggregation=aggregation, + event_partition_scopes=wrong_session_scopes, + ) snapshot = curated_module._write_curated_bars( tmp_path, bars, @@ -1049,6 +1101,8 @@ def test_event_bar_writer_partitions_independent_sources_without_rejecting_them( assert {item.relative_path for item in snapshot.partitions} == { item.relative_path for item in evidence } + with pytest.raises(ValidationError, match="event_count"): + load_verified_curated_bars(tmp_path, "multi-source-writer", snapshot.snapshot_id) def test_bound_reader_rejects_actual_consumed_bytes_that_do_not_match_manifest(