diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_index_clocks.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_index_clocks.py new file mode 100644 index 000000000..11b3afc13 --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_index_clocks.py @@ -0,0 +1,361 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Temporal queries must compare stored times and windows in the same clock.""" + +import datetime +import json + +import diskgraph +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from test_index_clocks import CLOCKS, DAY, EPOCH_DAY, TICK_NS + + +def make_graph(root, clock, ids, owners, times, *, other_times=None, other_dtype="timestamp"): + root.mkdir() + pq.write_table(pa.table({"id": [1, 2, 3]}), root / "users.parquet") + pq.write_table(pa.table({ + "id": ids, + "owner": owners, + "time": times, + "other_time": times if other_times is None else other_times, + "amount": [float(i + 1) for i in range(len(ids))], + "text": ["payload " * 20] * len(ids), + }), root / "events.parquet") + config = { + "dataset_name": "temporal_clocks", "num_shards": 2, + "timestamp_unit": clock, + "tables": [ + { + "name": "users", "primary_key": "id", + "columns": [{"name": "id", "ingested_dtype": "int64"}], + "source": {"type": "parquet", "paths": [str(root / "users.parquet")]}, + }, + { + "name": "events", "primary_key": "id", "create_time": "time", + "columns": [ + {"name": name, "ingested_dtype": dtype} + for name, dtype in [("id", "int64"), ("owner", "int64"), + ("time", "timestamp"), ("other_time", other_dtype), + ("amount", "float64"), ("text", "string")] + ], + "fkeys": [{"src_column": "owner", "dst_table": "users", "reverse": { + "time": {"table": "events", "column": "time", "unit": "s"} + }}], + "source": {"type": "parquet", "paths": [str(root / "events.parquet")]}, + }, + ], + } + diskgraph.ingest(json.dumps(config), str(root / "index"), col_stats=False, + context_threads=2, encode_threads=2, pk_map_threads=2, + edges_map_threads=2, index_threads=2) + engine = diskgraph.SamplerEngine(str(root / "index")) + relations = json.loads(engine.get_type_mapping())["edge_relations"] + relation = next(r["name"] for r in relations if r["src_type"] == "users") + return engine, relation + + +def plan_for(relation, *, time_column="time", start=-6 * DAY, limit=1, shared=False): + return { + "frames": [{"name": "f", "root_type": "users", "path": [ + {"relation": relation, "destination_type": "events"}], + "time_column": time_column, "start_seconds": start, "stop_seconds": 0, + "distinct": True, "columns": ["id", "amount", "text"]}], + "sequences": [{"name": "s", "frame": "f", "columns": ["id", "text"], + "order": "descending", "limit": limit}], + "aggregates": ([ + {"name": "n", "frame": "f", "operation": "count"}, + {"name": "total", "frame": "f", "operation": "sum", "column": "amount"}, + {"name": "age", "frame": "f", "operation": "recency", "column": time_column}, + ] if shared else []), + } + + +def values(value): + return value.to_pylist() if isinstance(value, pa.Array) else value + + +def run(engine, plan, owners, anchors, output_format="model"): + roots = {"users": engine.lookup_node_ids("users", pa.array(owners, type=pa.int64()))} + cutoffs = {"users": anchors} + result = engine.query_temporal(roots, cutoffs, json.dumps(plan), output_format=output_format) + lengths = engine.query_temporal_lengths(roots, cutoffs, json.dumps(plan)) + sequence = result["sequences"][0] + assert lengths["sequences"][0]["lengths"].to_pylist() == [ + b - a for a, b in zip(values(sequence["offsets"]), values(sequence["offsets"])[1:]) + ] + return result, lengths + + +@pytest.fixture(scope="module", params=CLOCKS) +def day_graph(tmp_path_factory, request): + root = tmp_path_factory.mktemp("temporal_clocks") / request.param + # Start-boundary rows, an exact timestamp tie, and one future row. + times = pa.array([(EPOCH_DAY + d) * DAY for d in (0, 1, 3, 3, 6, 9)], pa.timestamp("s")) + engine, relation = make_graph(root, request.param, [90, 80, 2, 10, 50, 60], [1] * 6, times) + return engine, relation, request.param + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +@pytest.mark.parametrize("time_column", ["time", "other_time"]) +@pytest.mark.parametrize("shared", [False, True]) +def test_all_clocks_select_the_same_rows(day_graph, output_format, time_column, shared): + engine, relation, _ = day_graph + plan = plan_for(relation, time_column=time_column, limit=2, shared=shared) + result, lengths = run(engine, plan, [1, 2, 1], + [(EPOCH_DAY + d) * DAY for d in (6, 6, 3)], output_format) + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == [50, 2, 2, 10] + assert values(sequence["offsets"]) == [0, 2, 2, 4] + assert values(sequence["timestamps"]) == [(EPOCH_DAY + d) * DAY for d in (6, 3, 3, 3)] + if shared: + aggregates = {a["name"]: values(a["values"]) for a in result["aggregates"]} + assert aggregates == {"n": [4, 0, 4], "total": [14.0, 0.0, 10.0], + "age": [0.0, None, 0.0]} + if time_column == "time": + assert lengths["stats"]["rowpack_timestamps"] == 0 + else: + assert lengths["stats"]["rowpack_timestamps"] > 0 + + +def test_unbounded_and_delivery_order(day_graph): + engine, relation, _ = day_graph + plan = plan_for(relation, start=None, limit=None) + plan["sequences"][0]["output_order"] = "ascending" + result, _ = run(engine, plan, [1], [(EPOCH_DAY + 6) * DAY]) + assert result["sequences"][0]["batch"].column("id").to_pylist() == [90, 80, 10, 2, 50] + plan["frames"][0]["stop_seconds"] = None + result, _ = run(engine, plan, [1], [(EPOCH_DAY + 6) * DAY]) + assert result["sequences"][0]["batch"].column("id").to_pylist() == [90, 80, 10, 2, 50, 60] + + +def test_future_window_aggregate_labels(day_graph): + engine, relation, _ = day_graph + plan = plan_for(relation, start=0, limit=2, shared=True) + plan["frames"][0]["stop_seconds"] = 6 * DAY + result, _ = run(engine, plan, [1, 1], [(EPOCH_DAY + d) * DAY for d in (6, 0)]) + assert result["sequences"][0]["batch"].column("id").to_pylist() == [60, 50, 2] + aggregates = {a["name"]: values(a["values"]) for a in result["aggregates"]} + assert aggregates == {"n": [1, 4], "total": [6.0, 14.0], "age": [-3 * DAY, -6 * DAY]} + + +def test_recency_uses_the_same_rounded_anchor_as_filtering(day_graph): + engine, relation, clock = day_graph + plan = plan_for(relation, shared=True) + anchor = (EPOCH_DAY + 6) * DAY + 4000 + result, _ = run(engine, plan, [1], [anchor]) + rounded = anchor * 10**9 // TICK_NS[clock] * TICK_NS[clock] // 10**9 + assert values(result["aggregates"][2]["values"]) == [rounded - (EPOCH_DAY + 6) * DAY] + + +@pytest.mark.parametrize("shared", [False, True]) +def test_clock_support_preserves_selective_feature_decoding(day_graph, shared): + engine, relation, _ = day_graph + plan = plan_for(relation, start=None, limit=1, shared=shared) + result, _ = run(engine, plan, [1], [(EPOCH_DAY + 6) * DAY]) + assert result["sequences"][0]["batch"].column("id").to_pylist() == [50] + # Five rows qualify. A sole history decodes only its chosen row; a shared + # aggregate must read all five amounts, but only the chosen row's text. + assert result["stats"]["rows_decoded"] == (6 if shared else 1) + assert result["stats"]["values_decoded"] == (16 if shared else 4) + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +def test_multiple_windows_share_normalized_anchors(day_graph, output_format): + engine, relation, clock = day_graph + plan = plan_for(relation, limit=2, shared=True) + plan["frames"].append({**plan["frames"][0], "name": "future", + "start_seconds": 0, "stop_seconds": 6 * DAY}) + plan["sequences"].append({**plan["sequences"][0], "name": "future", "frame": "future"}) + plan["aggregates"] += [ + {**aggregate, "name": "future_" + aggregate["name"], "frame": "future"} + for aggregate in plan["aggregates"] + ] + anchors = [(EPOCH_DAY + 6) * DAY + 4000, (EPOCH_DAY + 6) * DAY, (EPOCH_DAY + 3) * DAY] + roots = {"users": engine.lookup_node_ids("users", pa.array([1, 2, 1]))} + cutoffs = {"users": anchors} + result = engine.query_temporal(roots, cutoffs, json.dumps(plan), output_format=output_format) + lengths = engine.query_temporal_lengths(roots, cutoffs, json.dumps(plan)) + for sequence, counts, ids, expected in zip( + result["sequences"], lengths["sequences"], + [[50, 2, 2, 10], [60, 60, 50]], [[2, 0, 2], [1, 0, 2]], + ): + assert sequence["batch"].column("id").to_pylist() == ids + assert counts["lengths"].to_pylist() == expected + rounded = anchors[0] * 10**9 // TICK_NS[clock] * TICK_NS[clock] // 10**9 + age = rounded - (EPOCH_DAY + 6) * DAY + assert {a["name"]: values(a["values"]) for a in result["aggregates"]} == { + "n": [4, 0, 4], "total": [14.0, 0.0, 10.0], "age": [age, None, 0], + "future_n": [1, 0, 2], "future_total": [6.0, 0.0, 11.0], + "future_age": [age - 3 * DAY, None, -6 * DAY], + } + + +@pytest.fixture(scope="module", params=CLOCKS[:3]) +def fine_graph(tmp_path_factory, request): + base = 1_700_000_000 * 10**9 + times = [base + n for n in (100_000_000, 900_000_000, 900_000_000, 1_100_000_000)] + times += [-1_900_000_000, -1_100_000_000, -900_000_000, 100_000_000] + root = tmp_path_factory.mktemp("fine_clocks") / request.param + return make_graph(root, request.param, [9, 2, 10, 20, 29, 22, 21, 30], + [1] * 4 + [2] * 4, pa.array(times, pa.timestamp("ns"))) + + +@pytest.mark.parametrize("time_column", ["time", "other_time"]) +@pytest.mark.parametrize("shared", [False, True]) +def test_subsecond_selection_and_negative_epochs(fine_graph, time_column, shared): + engine, relation = fine_graph + plan = plan_for(relation, time_column=time_column, start=-1, limit=1, shared=shared) + plan["sequences"][0]["columns"].append(time_column) + result, _ = run(engine, plan, [1, 2], [1_700_000_001, 0]) + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == [2, 21] + assert values(sequence["timestamps"]) == [1_700_000_000, -1] + column = sequence["batch"].column(time_column) + scale = {"ns": 10**9, "us": 10**6, "ms": 10**3}[column.type.unit] + assert column.cast(pa.int64()).to_pylist() == [ + 1_700_000_000 * scale + 9 * scale // 10, -9 * scale // 10] + if shared: + aggregates = {a["name"]: values(a["values"]) for a in result["aggregates"]} + assert aggregates["n"] == [3, 1] + assert aggregates["total"] == [6.0, 7.0] + assert aggregates["age"] == pytest.approx([0.1, 0.9]) + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +def test_timestamp_reductions_do_not_collapse_subseconds(fine_graph, output_format): + engine, relation = fine_graph + plan = plan_for(relation, start=-1, limit=None) + plan["aggregates"] = [ + {"name": "distinct", "frame": "f", "operation": "count_distinct", "column": "time"}, + {"name": "mode_count", "frame": "f", "groups": [{"keys": ["time"], "values": [ + {"name": "n", "operation": "count"}]}], "operation": "max", "column": "n"}, + {"name": "grouped_age", "frame": "f", "groups": [{"keys": ["owner"], "values": [ + {"name": "last", "operation": "max", "column": "time"}]}], + "operation": "recency", "column": "last"}, + ] + plan["aggregates"] += [ + {"name": operation, "frame": "f", "operation": operation, "column": "time"} + for operation in ("min", "max", "mode") + ] + plan["frames"][0]["columns"].append("owner") + result, _ = run(engine, plan, [1, 3], [1_700_000_001] * 2, output_format) + assert {a["name"]: values(a["values"]) for a in result["aggregates"]} == { + "distinct": [2, 0], "mode_count": [2, None], "grouped_age": [pytest.approx(0.1), None], + "min": [1_700_000_000, None], "max": [1_700_000_000, None], + "mode": [1_700_000_000, None]} + if output_format == "model": + for aggregate in result["aggregates"][-3:]: + assert aggregate["values"].type == pa.int64() + + +@pytest.mark.parametrize("method", ["query_temporal", "query_temporal_lengths"]) +@pytest.mark.parametrize("bound", ["start_seconds", "stop_seconds"]) +@pytest.mark.parametrize("seconds", [-1, 1]) +def test_subtick_windows_are_rejected(day_graph, method, bound, seconds): + engine, relation, clock = day_graph + if clock not in ("hours", "days"): + pytest.skip("a one-second bound is representable in this clock") + plan = plan_for(relation, start=None) + plan["frames"][0]["stop_seconds"] = None + plan["frames"][0][bound] = seconds + roots = {"users": engine.lookup_node_ids("users", pa.array([1]))} + with pytest.raises(diskgraph.DiskgraphConfigError, match="shorter than one tick"): + getattr(engine, method)(roots, {"users": [(EPOCH_DAY + 6) * DAY]}, json.dumps(plan)) + + +@pytest.fixture(scope="module", params=["hours", "days"]) +def integer_graph(tmp_path_factory, request): + clock = request.param + tick = TICK_NS[clock] // 10**9 + base = EPOCH_DAY * DAY + times = [base + n * tick // 2 for n in (20, 21, 22, 23)] + times += [n * tick // 2 for n in (-5, -4, -3, -2, -1, 0)] + engine, relation = make_graph( + tmp_path_factory.mktemp("integer_clocks") / clock, clock, + [1, 30, 2, 4, 20, 21, 22, 23, 24, 25], [1] * 4 + [2] * 6, + pa.array(times, pa.timestamp("s")), other_times=pa.array(times, pa.int64()), other_dtype="int64") + return engine, relation, tick, times + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +@pytest.mark.parametrize("shared", [False, True]) +@pytest.mark.parametrize("case", ["past", "fractional_start", "fractional_stop", "negative"]) +def test_integer_times_follow_ingest_rounding(integer_graph, output_format, shared, case): + engine, relation, tick, times = integer_graph + base = EPOCH_DAY * DAY + # Hand-calculated answers: event times round up, while anchors and relative + # durations truncate towards zero, as in ingestion and the graph/TT paths. + owner, anchor, start, stop, expected = { + "past": (1, base + 10 * tick + tick // 4, None, 0, [1]), + "fractional_start": (1, base + 11 * tick + tick // 4, -3 * tick // 2, 0, [2, 30]), + "fractional_stop": (1, base + 10 * tick + tick // 4, None, 3 * tick // 2, [2, 30, 1]), + "negative": (2, -7 * tick // 4, None, 0, [23, 22, 21, 20]), + }[case] + plan = plan_for(relation, time_column="other_time", start=start, limit=None, shared=shared) + plan["frames"][0]["stop_seconds"] = stop + result, _ = run(engine, plan, [owner], [anchor], output_format) + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == expected + event_times = dict(zip([1, 30, 2, 4, 20, 21, 22, 23, 24, 25], times)) + assert values(sequence["timestamps"]) == [event_times[node] for node in expected] + if shared: + amounts = {node: i + 1 for i, node in enumerate(event_times)} + aggregates = {a["name"]: values(a["values"])[0] for a in result["aggregates"]} + rounded_anchor = -tick if case == "negative" else base + ( + 11 if case == "fractional_start" else 10) * tick + assert aggregates == {"n": len(expected), "total": sum(amounts[node] for node in expected), + "age": rounded_anchor - max(event_times[node] for node in expected)} + # A sole bounded sequence must agree with the full sequence prefix. This + # also checks the selection after applying a limit. + plan["sequences"][0]["limit"] = 1 + limited, _ = run(engine, plan, [owner], [anchor], output_format) + assert limited["sequences"][0]["batch"].column("id").to_pylist() == expected[:1] + assert values(limited["sequences"][0]["timestamps"]) == [event_times[expected[0]]] + # The declared clock column contains the same source instants, converted by + # ingestion. Indexed filtering must admit the same rows; rounding can make + # additional time ties, so their delivery orders need not match. + plan["aggregates"] = [] + plan["sequences"][0]["limit"] = None + plan["frames"][0]["time_column"] = "time" + indexed, _ = run(engine, plan, [owner], [anchor], output_format) + assert sorted(indexed["sequences"][0]["batch"].column("id").to_pylist()) == sorted(expected) + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +def test_sequence_metadata_preserves_dates_beyond_nanosecond_range(tmp_path, output_format): + engine, relation = make_graph( + tmp_path / "date_clocks", "nanoseconds", [2, 1], [1, 1], + pa.array([EPOCH_DAY * DAY] * 2, pa.timestamp("s")), + other_times=pa.array([datetime.date(3000, 1, 1), datetime.date(9999, 12, 31)], pa.date32()), + other_dtype="date") + plan = plan_for(relation, time_column="other_time", start=None) + plan["frames"][0]["stop_seconds"] = None + result, _ = run(engine, plan, [1], [EPOCH_DAY * DAY], output_format) + assert result["sequences"][0]["batch"].column("id").to_pylist() == [1] + assert values(result["sequences"][0]["timestamps"]) == [253_402_214_400] + + +@pytest.mark.parametrize("clock", ["minutes", "weeks", "months", "years"]) +def test_unsupported_index_clock_is_a_configuration_error(tmp_path, clock): + # An integer-only index can be opened even with an unsupported clock. + source = tmp_path / "users.parquet" + pq.write_table(pa.table({"id": [1]}), source) + diskgraph.ingest(json.dumps({"dataset_name": "invalid_clock", "tables": [{ + "name": "users", "primary_key": "id", + "columns": [{"name": "id", "ingested_dtype": "int64"}], + "source": {"type": "parquet", "paths": [str(source)]}, + }]}), str(tmp_path / "index"), col_stats=False, + context_threads=2, encode_threads=2, pk_map_threads=2, edges_map_threads=2, index_threads=2) + metadata_path = tmp_path / "index" / "index_metadata.json" + metadata = json.loads(metadata_path.read_text()) + metadata["timestamp_unit"] = clock + metadata_path.write_text(json.dumps(metadata)) + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + for method in ("query_temporal", "query_temporal_lengths"): + with pytest.raises(diskgraph.DiskgraphConfigError, match="not an index clock"): + getattr(engine, method)({"users": [0]}, {"users": [0]}, '{"frames": []}') diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index be9ba21af..fa7c13319 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -24,6 +24,8 @@ use arrow::pyarrow::ToPyArrow; use arrow_array::RecordBatch; use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; +use diskgraph_core::time::{convert_ceil, convert_floor, split_parts, Abs, Dur, TimeUnit, UnitSystem}; +use diskgraph_core::with_index_clock; use diskgraph_engine::edge_unpacker::EdgeUnpacker; use diskgraph_index::graph::{GraphIndexReader, TableReader}; use pyo3::prelude::*; @@ -149,6 +151,7 @@ enum ScalarValue { U64(u64), F64(f64), String(String), + Timestamp(i64, TimeUnit), } /// Python `repr(float)`: Rust's shortest round-trip digits and exponent cutoffs, Python spelling. @@ -221,6 +224,10 @@ impl ScalarValue { Self::I64(value) => Ok(Some(*value as f64)), Self::U64(value) => Ok(Some(*value as f64)), Self::F64(value) => Ok(Some(*value)), + Self::Timestamp(value, unit) => { + let (seconds, nanos) = split_parts(*value, *unit); + Ok(Some(seconds as f64 + f64::from(nanos) / 1e9)) + } Self::Bool(_) | Self::String(_) => Err("reduction requires a numeric field".into()), } } @@ -232,10 +239,34 @@ impl ScalarValue { Self::U64(value) => i64::try_from(*value) .map(Some) .map_err(|_| "timestamp exceeds int64 range".to_string()), + Self::Timestamp(value, unit) => { + Ok(Some(convert_floor(*value, *unit, TimeUnit::Seconds))) + } _ => Err("time field must be an integer or timestamp".to_string()), } } + fn index_time(&self) -> Result, String> { + match self { + Self::Timestamp(value, unit) => Ok(Some(convert_ceil(*value, *unit, U::UNIT))), + _ => self + .epoch_seconds() + .map(|value| value.map(|value| convert_ceil(value, TimeUnit::Seconds, U::UNIT))), + } + } + + // A wide integer preserves timestamp precision across units without overflowing. + fn epoch_nanos(&self) -> Result, String> { + match self { + Self::Timestamp(value, unit) => { + Ok(Some(i128::from(*value) * i128::from(unit.tick_nanos()))) + } + _ => self + .epoch_seconds() + .map(|value| value.map(|value| i128::from(value) * 1_000_000_000)), + } + } + fn equality_key(&self) -> String { match self { Self::Null => "null".to_string(), @@ -244,6 +275,10 @@ impl ScalarValue { Self::U64(value) => format!("int:{value}"), Self::F64(value) => format!("float:{:016x}", value.to_bits()), Self::String(value) => format!("str:{value}"), + Self::Timestamp(value, unit) => format!( + "timestamp:{}", + i128::from(*value) * i128::from(unit.tick_nanos()) + ), } } @@ -256,6 +291,10 @@ impl ScalarValue { Self::U64(value) => ("int".into(), value.to_string()), Self::F64(value) => ("float".into(), python_float_repr(*value)), Self::String(value) => ("str".into(), python_str_repr(value)), + Self::Timestamp(value, unit) => ( + "int".into(), + (i128::from(*value) * i128::from(unit.tick_nanos())).to_string(), + ), } } @@ -270,6 +309,9 @@ impl ScalarValue { (Self::U64(left), Self::I64(right)) => Ok(i128::from(*left).cmp(&i128::from(*right))), (Self::String(left), Self::String(right)) => Ok(left.cmp(right)), (Self::Bool(left), Self::Bool(right)) => Ok(left.cmp(right)), + (Self::Timestamp(..), Self::Timestamp(..)) => { + Ok(self.epoch_nanos()?.cmp(&other.epoch_nanos()?)) + } _ => Err("min/max values have incompatible types".to_string()), } } @@ -282,6 +324,10 @@ impl ScalarValue { Self::U64(value) => list.append(*value), Self::F64(value) => list.append(*value), Self::String(value) => list.append(value), + Self::Timestamp(..) => list.append( + self.epoch_seconds() + .map_err(DiskgraphConfigError::new_err)?, + ), } } } @@ -308,12 +354,13 @@ fn scalar_values_to_array( }) .collect::, _>>()?, ))), - Some(ScalarValue::I64(_)) => Ok(Arc::new(Int64Array::from( + Some(ScalarValue::I64(_) | ScalarValue::Timestamp(..)) => Ok(Arc::new(Int64Array::from( values .iter() .map(|value| match value { ScalarValue::Null => Ok(None), ScalarValue::I64(value) => Ok(Some(*value)), + ScalarValue::Timestamp(..) => value.epoch_seconds(), _ => Err("aggregate values have inconsistent physical types".to_string()), }) .collect::, _>>()?, @@ -516,24 +563,37 @@ fn scalar_at(array: &dyn Array, row: usize) -> Result { return Ok(ScalarValue::String(values.value(row).to_string())); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64(values.value(row))); + return Ok(ScalarValue::Timestamp(values.value(row), TimeUnit::Seconds)); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64(values.value(row).div_euclid(1_000))); + return Ok(ScalarValue::Timestamp( + values.value(row), + TimeUnit::Milliseconds, + )); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64(values.value(row).div_euclid(1_000_000))); + return Ok(ScalarValue::Timestamp( + values.value(row), + TimeUnit::Microseconds, + )); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64( - values.value(row).div_euclid(1_000_000_000), + return Ok(ScalarValue::Timestamp( + values.value(row), + TimeUnit::Nanoseconds, )); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64(i64::from(values.value(row)) * 86_400)); + return Ok(ScalarValue::Timestamp( + i64::from(values.value(row)), + TimeUnit::Days, + )); } if let Some(values) = array.as_any().downcast_ref::() { - return Ok(ScalarValue::I64(values.value(row).div_euclid(1_000))); + return Ok(ScalarValue::Timestamp( + values.value(row), + TimeUnit::Milliseconds, + )); } Err(format!( "unsupported scalar Arrow type {}", @@ -779,7 +839,7 @@ fn take_batch(batch: &RecordBatch, rows: &[u64], columns: &[usize]) -> Result, ScalarValue)]) { +fn sort_frame_rows(rows: &mut [(usize, Option, ScalarValue, Option)]) { rows.sort_by_key(|row| row.1); let mut start = 0; while start < rows.len() { @@ -795,18 +855,44 @@ fn sort_frame_rows(rows: &mut [(usize, Option, ScalarValue)]) { } } -fn materialize_frame( +type TimeWindow = (Option, Option); + +/// The wire plan uses seconds; every comparison within a frame uses index ticks. +fn frame_windows( + plan: &FramePlan, + cutoffs: &[Abs], +) -> Result, String> { + let duration = |seconds| { + crate::tt_builder::check_tick::("window bound", seconds, TimeUnit::Seconds)?; + Ok::<_, String>(Dur::::from_secs(seconds).raw()) + }; + let lower = plan.start_seconds.map(duration).transpose()?; + let upper = plan.stop_seconds.map(duration).transpose()?; + Ok(cutoffs + .iter() + .map(|cutoff| { + let cutoff = cutoff.raw(); + ( + lower.map(|offset| cutoff.saturating_add(offset)), + upper.map(|offset| cutoff.saturating_add(offset)), + ) + }) + .collect()) +} + +fn materialize_frame( reader: &GraphIndexReader, plan: &FramePlan, root_type: &str, roots: &[u64], - cutoffs: &[i64], + cutoffs: &[Abs], sequence: Option<&SequencePlan>, full_columns: Option<(&HashSet<&str>, usize)>, timings: &mut QueryTimings, stats: &mut QueryStats, ) -> Result { validate_frame_plan(plan, root_type)?; + let windows = frame_windows::(plan, cutoffs)?; let (hops, destination) = frame_path(reader, plan, root_type)?; let traversal_started = Instant::now(); @@ -830,10 +916,8 @@ fn materialize_frame( let window = decoder.as_ref().map(|decoder| { ( decoder, - plan.start_seconds - .map(|seconds| cutoffs[anchor].saturating_add(seconds)), - plan.stop_seconds - .map(|seconds| cutoffs[anchor].saturating_add(seconds)), + windows[anchor].0, + windows[anchor].1, plan.name.as_str(), ) }); @@ -932,24 +1016,17 @@ fn materialize_frame( let mut offsets = Vec::with_capacity(roots.len() + 1); offsets.push(0); for anchor in 0..roots.len() { - let lower = plan - .start_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); - let upper = plan - .stop_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); - let mut anchor_rows = Vec::<(usize, Option, ScalarValue)>::new(); + let (lower, upper) = windows[anchor]; + let mut anchor_rows = Vec::<(usize, Option, ScalarValue, Option)>::new(); for row in input_offsets[anchor]..input_offsets[anchor + 1] { - let timestamp = match time_position { - Some(position) => Some( - scalar_at(decoded.column(position).as_ref(), row)? - .epoch_seconds()? - .ok_or_else(|| { - format!("frame {:?} contains a null timestamp", plan.name) - })?, - ), - None => None, - }; + let time = time_position + .map(|position| scalar_at(decoded.column(position).as_ref(), row)) + .transpose()? + .unwrap_or(ScalarValue::Null); + let timestamp = time.index_time::()?; + if time_position.is_some() && timestamp.is_none() { + return Err(format!("frame {:?} contains a null timestamp", plan.name)); + } if timestamp.is_some_and(|value| { lower.is_some_and(|bound| value <= bound) || upper.is_some_and(|bound| value > bound) @@ -957,10 +1034,17 @@ fn materialize_frame( continue; } let key = scalar_at(decoded.column(primary_key_position).as_ref(), row)?; - anchor_rows.push((row, timestamp, key)); + let seconds = time.epoch_seconds()?; + // A frame has one time column/unit. Sort its raw values so clock + // rounding or saturation cannot create artificial timestamp ties. + let sort_time = match time { + ScalarValue::Timestamp(value, _) => Some(value), + _ => seconds, + }; + anchor_rows.push((row, sort_time, key, seconds)); } sort_frame_rows(&mut anchor_rows); - for (row, timestamp, _) in anchor_rows { + for (row, _, _, timestamp) in anchor_rows { selected_rows.push(row as u64); node_ids.push(flat_nodes[row]); timestamps.push(timestamp); @@ -1181,6 +1265,9 @@ fn reduce( let total = values.iter().try_fold(0i128, |total, value| { let value = match (sum_type, value) { (SumType::Int64, ScalarValue::I64(value)) => i128::from(*value), + (SumType::Int64, ScalarValue::Timestamp(value, unit)) => { + i128::from(convert_floor(*value, *unit, TimeUnit::Seconds)) + } (SumType::Uint64, ScalarValue::U64(value)) => i128::from(*value), _ => return Err("integer sum requires matching integer operands".into()), }; @@ -1201,13 +1288,17 @@ fn reduce( if operation == "recency" { let latest = values .iter() - .map(ScalarValue::epoch_seconds) + .map(ScalarValue::epoch_nanos) .collect::, _>>()? .into_iter() .flatten() .max() .expect("non-null timestamp values"); - return Ok(ScalarValue::F64(cutoff.saturating_sub(latest) as f64)); + let elapsed = (i128::from(cutoff) * 1_000_000_000 - latest).clamp( + i128::from(i64::MIN) * 1_000_000_000, + i128::from(i64::MAX) * 1_000_000_000, + ); + return Ok(ScalarValue::F64(elapsed as f64 / 1e9)); } if operation == "mode" { let mut counts = HashMap::::new(); @@ -1419,16 +1510,17 @@ fn aggregate_kind(frame: &MaterializedFrame, plan: &AggregatePlan) -> Option( reader: &GraphIndexReader, plan: &FramePlan, root_type: &str, roots: &[u64], - cutoffs: &[i64], + cutoffs: &[Abs], timings: &mut LengthTimings, stats: &mut LengthStats, ) -> Result, String> { validate_frame_plan(plan, root_type)?; + let windows = frame_windows::(plan, cutoffs)?; let (hops, destination) = frame_path(reader, plan, root_type)?; let final_sort_key = hops .last() @@ -1482,12 +1574,7 @@ fn count_frame( }); if let Some(decoder) = sort_decoder { for (anchor, rows) in rows_by_anchor.iter().enumerate() { - let lower = plan - .start_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); - let upper = plan - .stop_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); + let (lower, upper) = windows[anchor]; let mut selected = 0; for &(_, raw_sort_key) in rows { let timestamp = exact_key_time(&decoder, raw_sort_key) @@ -1514,16 +1601,11 @@ fn count_frame( let decoded = table_rows(table.as_ref(), &flat_nodes, &projection_indices)?; let mut row = 0; for (anchor, rows) in rows_by_anchor.iter().enumerate() { - let lower = plan - .start_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); - let upper = plan - .stop_seconds - .map(|value| cutoffs[anchor].saturating_add(value)); + let (lower, upper) = windows[anchor]; let mut selected = 0; for _ in rows { let timestamp = scalar_at(decoded.column(0).as_ref(), row)? - .epoch_seconds()? + .index_time::()? .ok_or_else(|| { format!("frame {:?} contains a null timestamp", plan.name) })?; @@ -1563,6 +1645,24 @@ pub(crate) fn execute_lengths( } let plan: QueryPlan = serde_json::from_str(plan_json) .map_err(|error| format!("invalid temporal query JSON: {error}"))?; + let clock = reader.metadata().timestamp_unit; + if !clock.is_index_clock() { + return Err(format!("{clock:?} is not an index clock")); + } + with_index_clock!(clock, U => { + let cutoffs = cutoffs.iter().copied().map(Abs::::from_secs).collect::>(); + execute_lengths_in_clock(reader, root_type, roots, &cutoffs, plan, total_started) + }) +} + +fn execute_lengths_in_clock( + reader: &GraphIndexReader, + root_type: &str, + roots: &[u64], + cutoffs: &[Abs], + plan: QueryPlan, + total_started: Instant, +) -> Result { let mut frame_plans = HashMap::new(); for frame in &plan.frames { if frame_plans.insert(frame.name.as_str(), frame).is_some() { @@ -1630,6 +1730,24 @@ pub(crate) fn execute( } let plan: QueryPlan = serde_json::from_str(plan_json) .map_err(|error| format!("invalid temporal query JSON: {error}"))?; + let clock = reader.metadata().timestamp_unit; + if !clock.is_index_clock() { + return Err(format!("{clock:?} is not an index clock")); + } + with_index_clock!(clock, U => { + let cutoffs = cutoffs.iter().copied().map(Abs::::from_secs).collect::>(); + execute_in_clock(reader, root_type, roots, &cutoffs, plan, total_started) + }) +} + +fn execute_in_clock( + reader: &GraphIndexReader, + root_type: &str, + roots: &[u64], + cutoffs: &[Abs], + plan: QueryPlan, + total_started: Instant, +) -> Result { for sequence in &plan.sequences { validate_sequence_plan(sequence)?; } @@ -1710,6 +1828,11 @@ pub(crate) fn execute( timings.sequence_assembly = elapsed_ns(sequence_started); let aggregate_started = Instant::now(); + let aggregate_cutoffs = if plan.aggregates.is_empty() { + Vec::new() + } else { + cutoffs.iter().map(|cutoff| cutoff.to_secs()).collect::>() + }; let aggregates = plan .aggregates .iter() @@ -1720,7 +1843,7 @@ pub(crate) fn execute( aggregate.name, aggregate.frame ) })?; - aggregate_result(frame, aggregate, cutoffs) + aggregate_result(frame, aggregate, &aggregate_cutoffs) }) .collect::, String>>()?; timings.aggregation = elapsed_ns(aggregate_started); @@ -1883,17 +2006,19 @@ mod tests { _ => Some((row % 7) as i64), }, keys[row % keys.len()].clone(), + Some(row as i64), ) }) .collect::>(); actual.shuffle(&mut rng); let mut expected = actual .iter() - .map(|(row, time, key)| (*row, *time, key.stable_sort_key())) + .map(|(row, time, key, _)| (*row, *time, key.stable_sort_key())) .collect::>(); expected .sort_by(|left, right| left.1.cmp(&right.1).then_with(|| left.2.cmp(&right.2))); sort_frame_rows(&mut actual); + assert!(actual.iter().all(|row| row.3 == Some(row.0 as i64))); assert_eq!( actual.iter().map(|row| row.0).collect::>(), expected.iter().map(|row| row.0).collect::>() diff --git a/packages/diskgraph/diskgraph-pybind/src/tt_builder.rs b/packages/diskgraph/diskgraph-pybind/src/tt_builder.rs index c839cd33e..8d5f701af 100644 --- a/packages/diskgraph/diskgraph-pybind/src/tt_builder.rs +++ b/packages/diskgraph/diskgraph-pybind/src/tt_builder.rs @@ -810,9 +810,11 @@ fn check_windows(op: &tt::Op) -> PyResult<()> { return Ok(()); } if *lo != UNBOUNDED_LOWER_OFFSET { - check_tick::("window bound", *lo, unit)?; + check_tick::("window bound", *lo, unit) + .map_err(crate::DiskgraphConfigError::new_err)?; } check_tick::("window bound", *hi, unit) + .map_err(crate::DiskgraphConfigError::new_err) } tt::Op::Autoregression { target_op, .. } | tt::Op::FutureLabels { target_op, .. } => { check_windows(target_op) @@ -821,13 +823,13 @@ fn check_windows(op: &tt::Op) -> PyResult<()> { } } -fn check_tick(what: &str, value: i64, unit: TimeUnit) -> PyResult<()> { +pub(crate) fn check_tick(what: &str, value: i64, unit: TimeUnit) -> Result<(), String> { if value != 0 && dur_from_unit::(value, unit).raw() == 0 { - return Err(crate::DiskgraphConfigError::new_err(format!( + return Err(format!( "a {what} of {value} {} is shorter than one tick of the {} index clock", unit_name(unit), unit_name(U::UNIT) - ))); + )); } Ok(()) } @@ -850,7 +852,8 @@ impl ShiftedOp { self.offsets .iter() .map(|&v| { - check_tick::("offset", v, self.unit)?; + check_tick::("offset", v, self.unit) + .map_err(crate::DiskgraphConfigError::new_err)?; Ok(dur_from_unit::(v, self.unit)) }) .collect() diff --git a/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs b/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs index f50d0dfa6..2cdb6d8db 100644 --- a/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs +++ b/packages/diskgraph/third_party/epserde/tests/test_review_regressions.rs @@ -1,4 +1,5 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. // SPDX-License-Identifier: Apache-2.0 use epserde::prelude::*; use std::ops::ControlFlow; diff --git a/scripts/license_header_policy.json b/scripts/license_header_policy.json index e40c0d7f4..cdbbcc80f 100644 --- a/scripts/license_header_policy.json +++ b/scripts/license_header_policy.json @@ -215,6 +215,258 @@ "kind": "vendored", "license": "MIT", "reason": "Vendored upstream PtrHash sources; preserve the upstream files and license." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/Cargo.toml", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/build.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/helpers.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/mem_case.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/read.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/reader_with_pos.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/deser/slice_with_pos.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/array.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/boxed_slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/iter.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/prim.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/stdlib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/string.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/tuple.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/impls/vec.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/lib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/helpers.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/write.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/ser/write_with_names.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/copy_type.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/traits/type_info.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/utils/aligned_cursor.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/src/utils/mod.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_bad_deser.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_bad_ser.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_generics.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_max_size_of.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_memcase.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_phantom.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_prim.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_slice.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_std.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_stdlib.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_tuples.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_type_hash_val.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_types.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." + }, + { + "pattern": "packages/diskgraph/third_party/epserde/tests/test_zero.rs", + "kind": "vendored", + "license": "Apache-2.0 OR LGPL-2.1-or-later", + "reason": "Copied epserde 0.8.0 source; retain upstream attribution and licensing recorded in packages/diskgraph/third_party/source-provenance.json." } ] } diff --git a/test/scripts/test_check_license_headers.py b/test/scripts/test_check_license_headers.py index 46ff5e561..1bf7f2ec0 100644 --- a/test/scripts/test_check_license_headers.py +++ b/test/scripts/test_check_license_headers.py @@ -297,3 +297,27 @@ def test_repository_candidate_tree_passes() -> None: ) assert errors == [] assert scanned > 3_000 + + +def test_epserde_vendor_boundary_is_exact() -> None: + prefix = "packages/diskgraph/third_party/epserde/" + exclusions = [entry for entry in POLICY["exclusions"] + if entry["pattern"].startswith(prefix)] + assert len(exclusions) == 42 + assert all(not any(marker in entry["pattern"] for marker in "*?[") + for entry in exclusions) + for relative in ("Cargo.toml", "src/deser/read.rs", "tests/test_types.rs"): + exclusion = check_license_headers.matching_exclusion( + prefix + relative, POLICY) + assert exclusion is not None + assert exclusion["kind"] == "vendored" + assert exclusion["license"] == "Apache-2.0 OR LGPL-2.1-or-later" + for relative in ("src/utils/alignment.rs", + "tests/test_review_regressions.rs", "new_source.rs"): + assert check_license_headers.matching_exclusion( + prefix + relative, POLICY) is None + assert check_license_headers.validate_text( + prefix + relative, SLASH_HEADER + "fn checked() {}\n", POLICY + ) is None + assert check_license_headers.validate_text( + prefix + relative, "fn unchecked() {}\n", POLICY) is not None