From 974ef52287c6d5414e57a14015bb17341cd0ef0e Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Wed, 4 Nov 2026 22:11:25 +0000 Subject: [PATCH 01/10] Read integer time columns in their declared unit in temporal queries An integer time column ingested with a timestamp_unit or timestamp_format was read as epoch seconds by the temporal query, while its edge sort key used the declared unit. Decode it with the same RowAccess path ingestion uses, for the window filter, lengths, returned timestamps and recency. Co-Authored-By: Claude Opus 5.5 --- packages/diskgraph/diskgraph-pybind/README.md | 6 +- .../diskgraph-pybind/src/bp_col_stats.rs | 2 +- .../diskgraph/diskgraph-pybind/src/lib.rs | 6 + .../diskgraph-pybind/src/temporal_query.rs | 197 +++++++++++++++--- 4 files changed, 176 insertions(+), 35 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/README.md b/packages/diskgraph/diskgraph-pybind/README.md index e56f24e55..135b4327c 100644 --- a/packages/diskgraph/diskgraph-pybind/README.md +++ b/packages/diskgraph/diskgraph-pybind/README.md @@ -263,8 +263,10 @@ result = engine.query_temporal( Windows are half-open relative to each aligned cutoff: `(cutoff + start_seconds, cutoff + stop_seconds]`. Either bound may be null; -bounds require `time_column`. Selected rows are sorted by timestamp and then -primary key before a sequence optionally reverses and truncates them. +bounds require `time_column`. An integer time column is read in the +`timestamp_unit` or `timestamp_format` it was ingested with, and `recency` +reads an integer column the same way. Selected rows are sorted by timestamp +and then primary key before a sequence optionally reverses and truncates them. `distinct` deduplicates destination dense IDs per root. Available reductions are `count`, `count_distinct`, `sum`, `mean`, `min`, diff --git a/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs b/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs index 20f25663a..9f5b1ed19 100644 --- a/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs +++ b/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs @@ -669,7 +669,7 @@ fn histogram_left_edge(spec: FloatingHistogramSpec, idx: usize) -> f64 { } } -fn row_access_hints(col: &ColumnSpec) -> RowAccessHints { +pub(crate) fn row_access_hints(col: &ColumnSpec) -> RowAccessHints { RowAccessHints { timestamp_format: col.timestamp_format.clone(), timestamp_unit: col.timestamp_unit.map(|unit| match unit { diff --git a/packages/diskgraph/diskgraph-pybind/src/lib.rs b/packages/diskgraph/diskgraph-pybind/src/lib.rs index e5c138b9a..7ce91e727 100644 --- a/packages/diskgraph/diskgraph-pybind/src/lib.rs +++ b/packages/diskgraph/diskgraph-pybind/src/lib.rs @@ -493,6 +493,7 @@ impl OutputHandler for PyOutputHandler { #[pyclass] pub struct SamplerEngine { reader: Arc, + time_hints: Arc, engine: Arc, pool: StatePool, sample_count: Arc, @@ -694,6 +695,7 @@ impl SamplerEngine { info!("SamplerEngine initialized successfully"); Ok(Self { reader, + time_hints: Arc::new(temporal_query::time_hints(root)), engine: Arc::new(engine), pool: StatePool::new(), sample_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), @@ -766,10 +768,12 @@ impl SamplerEngine { })?; let cutoffs: Vec = cutoff_values.extract()?; let reader = Arc::clone(&self.reader); + let time_hints = Arc::clone(&self.time_hints); let result = py .detach(move || { temporal_query::execute( reader.as_ref(), + time_hints.as_ref(), &root_type, &roots, &cutoffs, @@ -811,10 +815,12 @@ impl SamplerEngine { })?; let cutoffs: Vec = cutoff_values.extract()?; let reader = Arc::clone(&self.reader); + let time_hints = Arc::clone(&self.time_hints); let result = py .detach(move || { temporal_query::execute_lengths( reader.as_ref(), + time_hints.as_ref(), &root_type, &roots, &cutoffs, diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index be9ba21af..f5ed1645c 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -11,6 +11,7 @@ use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::Path; use std::sync::{Arc, LazyLock}; use std::time::Instant; @@ -26,6 +27,10 @@ use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; use diskgraph_engine::edge_unpacker::EdgeUnpacker; use diskgraph_index::graph::{GraphIndexReader, TableReader}; +use diskgraph_ingest::graph_config::{ + read_persisted_config, IngestedType, DISKGRAPH_CONFIG_FILENAME, +}; +use diskgraph_rowpack::{RowAccess, RowAccessHints, RowAccessStats}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; use regex::Regex; @@ -541,6 +546,79 @@ fn scalar_at(array: &dyn Array, row: usize) -> Result { )) } +/// Ingest hints by column name, for one table. +type ColumnHints = HashMap; + +/// Ingest hints of integer columns declared with a time unit or format, by table. +pub(crate) type TimeHints = HashMap; + +/// Reads [`TimeHints`] from the index's persisted graph config. Indexes without +/// one read integer times as epoch seconds. +pub(crate) fn time_hints(root: &Path) -> TimeHints { + if !root.join(DISKGRAPH_CONFIG_FILENAME).exists() { + return TimeHints::new(); + } + let config = match read_persisted_config(root) { + Ok(config) => config, + Err(error) => { + tracing::warn!("temporal queries read integer times as epoch seconds: {error:#}"); + return TimeHints::new(); + } + }; + config + .tables + .iter() + .map(|table| { + let hints = table + .columns + .iter() + .filter(|column| { + (column.timestamp_unit.is_some() || column.timestamp_format.is_some()) + && matches!( + column.ingested_dtype, + IngestedType::Int8 + | IngestedType::Int16 + | IngestedType::Int32 + | IngestedType::Int64 + | IngestedType::UInt8 + | IngestedType::UInt16 + | IngestedType::UInt32 + | IngestedType::UInt64 + ) + }) + .map(|column| { + ( + column.name.clone(), + crate::bp_col_stats::row_access_hints(column), + ) + }) + .collect(); + (table.name.clone(), hints) + }) + .collect() +} + +/// Epoch seconds of each value. A hinted integer column decodes exactly as +/// ingestion and its edge sort keys read it. +fn epoch_seconds( + array: &ArrayRef, + hint: Option<&RowAccessHints>, +) -> Result>, String> { + let Some(hint) = hint else { + return (0..array.len()) + .map(|row| scalar_at(array.as_ref(), row)?.epoch_seconds()) + .collect(); + }; + let mut access = RowAccess::new(vec![hint.clone()], RowAccessStats::new(None)); + access.set_cols(std::slice::from_ref(array)); + (0..array.len()) + .map(|row| { + access.set_row(row); + access.get_timestamp(0).map_err(|error| error.to_string()) + }) + .collect() +} + fn relation_columns( table: &TableReader, relation_name: &str, @@ -797,6 +875,7 @@ fn sort_frame_rows(rows: &mut [(usize, Option, ScalarValue)]) { fn materialize_frame( reader: &GraphIndexReader, + hints: &TimeHints, plan: &FramePlan, root_type: &str, roots: &[u64], @@ -925,7 +1004,14 @@ fn materialize_frame( input_offsets.push(input_offsets.last().copied().unwrap_or(0) + rows.len()); } let primary_key_position = positions[&primary_key]; - let time_position = plan.time_column.as_ref().map(|column| positions[column]); + let times = plan + .time_column + .as_ref() + .map(|column| { + let hint = hints.get(&destination).and_then(|hints| hints.get(column)); + epoch_seconds(decoded.column(positions[column]), hint) + }) + .transpose()?; let mut selected_rows = Vec::::new(); let mut node_ids = Vec::new(); let mut timestamps = Vec::new(); @@ -940,16 +1026,13 @@ fn materialize_frame( .map(|value| cutoffs[anchor].saturating_add(value)); let mut anchor_rows = Vec::<(usize, Option, ScalarValue)>::new(); for row in input_offsets[anchor]..input_offsets[anchor + 1] { - let timestamp = match time_position { - Some(position) => Some( - scalar_at(decoded.column(position).as_ref(), row)? - .epoch_seconds()? - .ok_or_else(|| { - format!("frame {:?} contains a null timestamp", plan.name) - })?, - ), - None => None, - }; + let timestamp = times + .as_ref() + .map(|times| { + times[row] + .ok_or_else(|| format!("frame {:?} contains a null timestamp", plan.name)) + }) + .transpose()?; if timestamp.is_some_and(|value| { lower.is_some_and(|bound| value <= bound) || upper.is_some_and(|bound| value > bound) @@ -1124,6 +1207,13 @@ impl RowsView<'_> { } } +fn column_hint<'a>( + hints: Option<&'a ColumnHints>, + column: Option<&str>, +) -> Option<&'a RowAccessHints> { + hints?.get(column?) +} + fn validate_reduction(operation: &str, column: Option<&str>) -> Result<(), String> { if !REDUCTIONS.contains(&operation) { return Err(format!("unsupported temporal reduction {operation:?}")); @@ -1140,6 +1230,7 @@ fn reduce( column: Option<&str>, cutoff: i64, sum_type: Option, + hint: Option<&RowAccessHints>, ) -> Result { validate_reduction(operation, column)?; if sum_type.is_some() && operation != "sum" { @@ -1199,10 +1290,14 @@ fn reduce( } } if operation == "recency" { - let latest = values + let mut seconds = values .iter() .map(ScalarValue::epoch_seconds) - .collect::, _>>()? + .collect::, _>>()?; + if hint.is_some() { + seconds = epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef), hint)?; + } + let latest = seconds .into_iter() .flatten() .max() @@ -1270,7 +1365,13 @@ fn reduce( }) } -fn group_rows(rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64) -> Result, String> { +/// `hints` apply only to frame columns, not to reductions of an earlier group. +fn group_rows( + rows: RowsView<'_>, + plan: &GroupPlan, + cutoff: i64, + hints: Option<&ColumnHints>, +) -> Result, String> { if plan.keys.is_empty() || plan.values.is_empty() { return Err("group_by requires keys and values".into()); } @@ -1337,6 +1438,7 @@ fn group_rows(rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64) -> Result, plan: &GroupPlan, cutoff: i64) -> Result, plan: &AggregatePlan, cutoffs: &[i64], ) -> Result { @@ -1359,9 +1462,9 @@ fn aggregate_result( stop: frame.offsets[anchor + 1], }; let value = if let Some((first, remaining)) = plan.groups.split_first() { - let mut grouped = group_rows(frame_view, first, cutoff)?; + let mut grouped = group_rows(frame_view, first, cutoff, hints)?; for group in remaining { - grouped = group_rows(RowsView::Owned(&grouped), group, cutoff)?; + grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, None)?; } reduce( RowsView::Owned(&grouped), @@ -1369,6 +1472,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, + None, )? } else { reduce( @@ -1377,6 +1481,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, + column_hint(hints, plan.column.as_deref()), )? }; values.push(value); @@ -1421,6 +1526,7 @@ fn aggregate_kind(frame: &MaterializedFrame, plan: &AggregatePlan) -> Option>(); let decoded = table_rows(table.as_ref(), &flat_nodes, &projection_indices)?; + let hint = hints + .get(&destination) + .and_then(|hints| hints.get(time_column)); + let times = epoch_seconds(decoded.column(0), hint)?; let mut row = 0; for (anchor, rows) in rows_by_anchor.iter().enumerate() { let lower = plan @@ -1522,11 +1632,9 @@ fn count_frame( .map(|value| cutoffs[anchor].saturating_add(value)); let mut selected = 0; for _ in rows { - let timestamp = scalar_at(decoded.column(0).as_ref(), row)? - .epoch_seconds()? - .ok_or_else(|| { - format!("frame {:?} contains a null timestamp", plan.name) - })?; + let timestamp = times[row].ok_or_else(|| { + format!("frame {:?} contains a null timestamp", plan.name) + })?; row += 1; stats.rowpack_timestamps += 1; if lower.is_none_or(|bound| timestamp > bound) @@ -1548,6 +1656,7 @@ fn count_frame( pub(crate) fn execute_lengths( reader: &GraphIndexReader, + hints: &TimeHints, root_type: &str, roots: &[u64], cutoffs: &[i64], @@ -1584,6 +1693,7 @@ pub(crate) fn execute_lengths( })?; let lengths = count_frame( reader, + hints, frame, root_type, roots, @@ -1615,6 +1725,7 @@ pub(crate) fn execute_lengths( pub(crate) fn execute( reader: &GraphIndexReader, + hints: &TimeHints, root_type: &str, roots: &[u64], cutoffs: &[i64], @@ -1679,6 +1790,7 @@ pub(crate) fn execute( }); let frame = materialize_frame( reader, + hints, frame_plan, root_type, roots, @@ -1720,7 +1832,7 @@ pub(crate) fn execute( aggregate.name, aggregate.frame ) })?; - aggregate_result(frame, aggregate, cutoffs) + aggregate_result(frame, hints.get(&frame.destination), aggregate, cutoffs) }) .collect::, String>>()?; timings.aggregation = elapsed_ns(aggregate_started); @@ -1961,7 +2073,7 @@ mod tests { fn reduction_empty_and_null_semantics_are_distinct() { let empty = Vec::::new(); assert!(matches!( - reduce(RowsView::Owned(&empty), "sum", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&empty), "sum", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(0.0) )); assert!(matches!( @@ -1970,7 +2082,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .unwrap(), ScalarValue::I64(0) @@ -1981,7 +2094,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .unwrap(), ScalarValue::Null @@ -1992,6 +2106,7 @@ mod tests { "count_distinct", Some("value"), 0, + None, None ) .unwrap(), @@ -2006,7 +2121,15 @@ mod tests { row(Some(ScalarValue::I64(1))), ]; assert!(matches!( - reduce(RowsView::Owned(&rows), "sum", Some("value"), 0, Some(SumType::Int64)).unwrap(), + reduce( + RowsView::Owned(&rows), + "sum", + Some("value"), + 0, + Some(SumType::Int64), + None + ) + .unwrap(), ScalarValue::I64(value) if value == (1 << 53) + 1 )); let output = scalar_values_to_array( @@ -2043,7 +2166,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .is_err()); let unsigned = [row(Some(ScalarValue::U64(u64::MAX)))]; @@ -2053,7 +2177,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Uint64) + Some(SumType::Uint64), + None ) .unwrap(), ScalarValue::U64(u64::MAX) @@ -2063,7 +2188,15 @@ mod tests { row(Some(ScalarValue::F64(2.5))), ]; assert!(matches!( - reduce(RowsView::Owned(&floats), "sum", Some("value"), 0, None).unwrap(), + reduce( + RowsView::Owned(&floats), + "sum", + Some("value"), + 0, + None, + None + ) + .unwrap(), ScalarValue::F64(3.75) )); } @@ -2076,11 +2209,11 @@ mod tests { row(Some(ScalarValue::F64(30.0))), ]; assert!(matches!( - reduce(RowsView::Owned(&rows), "var_population", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&rows), "var_population", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(value) if (value - 66.66666666666667).abs() < 1e-10 )); assert!(matches!( - reduce(RowsView::Owned(&rows), "std_sample", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&rows), "std_sample", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(value) if (value - 10.0).abs() < 1e-10 )); let tied = vec![ @@ -2088,7 +2221,7 @@ mod tests { row(Some(ScalarValue::String("a".into()))), ]; assert!(matches!( - reduce(RowsView::Owned(&tied), "mode", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&tied), "mode", Some("value"), 0, None, None).unwrap(), ScalarValue::String(value) if value == "a" )); } From 62662cbf8f1e19a76676a1b345386476ba34d127 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Wed, 4 Nov 2026 22:11:25 +0000 Subject: [PATCH 02/10] Test temporal queries over integer time units and formats Co-Authored-By: Claude Opus 5.5 --- .../tests/test_temporal_int_time_units.py | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py new file mode 100644 index 000000000..2f13fc8f4 --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integer time columns are read in their declared unit or format.""" + +import json +from datetime import UTC, datetime + +import diskgraph +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +CUTOFF = int(datetime(2024, 3, 10, tzinfo=UTC).timestamp()) +DAY = 86_400 +# Seconds before the cutoff; the window keeps (-3 days, 0]. +OFFSETS = [5 * DAY, 2 * DAY, DAY, 0, -DAY] +EXPECTED_IDS = [4, 3, 2] + + +def encode(seconds, hint): + if "timestamp_format" in hint: + return int(datetime.fromtimestamp(seconds, UTC).strftime("%Y%m%d")) + scale = {"seconds": 1, "millis": 10**3, "micros": 10**6, "nanos": 10**9} + return seconds * scale[hint["timestamp_unit"]] + + +@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) +@pytest.mark.parametrize( + "hint", + [ + {"timestamp_unit": "seconds"}, + {"timestamp_unit": "millis"}, + {"timestamp_unit": "micros"}, + {"timestamp_unit": "nanos"}, + {"timestamp_format": "%Y%m%d"}, + ], + ids=["s", "ms", "us", "ns", "yyyymmdd"], +) +def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): + times = [CUTOFF - offset for offset in OFFSETS] + pq.write_table(pa.table({"id": [1]}), tmp_path / "users.parquet") + pq.write_table( + pa.table( + { + "id": list(range(1, len(times) + 1)), + "owner": [1] * len(times), + "time": pa.array(times, type=pa.timestamp("s")), + "event_time": pa.array([encode(t, hint) for t in times], pa.int64()), + } + ), + tmp_path / "events.parquet", + ) + config = { + "dataset_name": "int_time_units", + "num_shards": 1, + "tables": [ + { + "name": "users", + "primary_key": "id", + "columns": [{"name": "id", "ingested_dtype": "int64"}], + "source": { + "type": "parquet", + "paths": [str(tmp_path / "users.parquet")], + }, + }, + { + "name": "events", + "primary_key": "id", + "create_time": "time", + "columns": [ + {"name": "id", "ingested_dtype": "int64"}, + {"name": "owner", "ingested_dtype": "int64"}, + {"name": "time", "ingested_dtype": "timestamp"}, + {"name": "event_time", "ingested_dtype": "int64", **hint}, + ], + "fkeys": [ + { + "src_column": "owner", + "dst_table": "users", + "reverse": { + "time": { + "table": "events", + "column": "event_time" if sort_key else "time", + "unit": "s", + } + }, + } + ], + "source": { + "type": "parquet", + "paths": [str(tmp_path / "events.parquet")], + }, + }, + ], + } + diskgraph.ingest( + json.dumps(config), + str(tmp_path / "index"), + col_stats=False, + context_threads=1, + encode_threads=1, + pk_map_threads=1, + edges_map_threads=1, + index_threads=1, + ) + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + relation = next( + relation["name"] + for relation in json.loads(engine.get_type_mapping())["edge_relations"] + if relation["src_type"] == "users" + ) + plan = json.dumps( + { + "frames": [ + { + "name": "f", + "root_type": "users", + "path": [{"relation": relation, "destination_type": "events"}], + "time_column": "event_time", + "start_seconds": -3 * DAY, + "stop_seconds": 0, + "distinct": True, + "columns": ["id"], + } + ], + "sequences": [ + { + "name": "s", + "frame": "f", + "columns": ["id"], + "order": "descending", + "limit": None, + } + ], + "aggregates": [ + { + "name": "r", + "frame": "f", + "operation": "recency", + "column": "event_time", + } + ], + } + ) + roots = {"users": engine.lookup_node_ids("users", pa.array([1], pa.int64()))} + cutoffs = {"users": [CUTOFF]} + + result = engine.query_temporal(roots, cutoffs, plan, output_format="inspectable") + lengths = engine.query_temporal_lengths(roots, cutoffs, plan) + + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == EXPECTED_IDS + assert list(sequence["timestamps"]) == [times[i - 1] for i in EXPECTED_IDS] + assert lengths["sequences"][0]["lengths"].to_pylist() == [len(EXPECTED_IDS)] + path = "sort_key_timestamps" if sort_key else "rowpack_timestamps" + assert lengths["stats"][path] == len(times) + assert list(result["aggregates"][0]["values"]) == [0.0] From 0bbf6a2c85201d9b2fd83b68c7a33e0fcfd1309f Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Wed, 4 Nov 2026 22:24:08 +0000 Subject: [PATCH 03/10] Carry integer time hints through temporal group stages recency over a group key, or over a min, max or mode of a hinted integer column, read the raw integer as seconds after a group stage. Co-Authored-By: Claude Opus 5.5 --- packages/diskgraph/diskgraph-pybind/README.md | 8 ++-- .../diskgraph-pybind/src/temporal_query.rs | 39 +++++++++++++++---- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/README.md b/packages/diskgraph/diskgraph-pybind/README.md index 135b4327c..b9218398c 100644 --- a/packages/diskgraph/diskgraph-pybind/README.md +++ b/packages/diskgraph/diskgraph-pybind/README.md @@ -264,9 +264,11 @@ result = engine.query_temporal( Windows are half-open relative to each aligned cutoff: `(cutoff + start_seconds, cutoff + stop_seconds]`. Either bound may be null; bounds require `time_column`. An integer time column is read in the -`timestamp_unit` or `timestamp_format` it was ingested with, and `recency` -reads an integer column the same way. Selected rows are sorted by timestamp -and then primary key before a sequence optionally reverses and truncates them. +`timestamp_unit` or `timestamp_format` it was ingested with. `recency` reads +such a column, or a group key or `min`/`max`/`mode` of it, the same way; +other reductions and projections return the stored values. Selected rows are +sorted by timestamp and then primary key before a sequence optionally reverses +and truncates them. `distinct` deduplicates destination dense IDs per root. Available reductions are `count`, `count_distinct`, `sum`, `mean`, `min`, diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index f5ed1645c..b5cc121fe 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -1008,7 +1008,7 @@ fn materialize_frame( .time_column .as_ref() .map(|column| { - let hint = hints.get(&destination).and_then(|hints| hints.get(column)); + let hint = column_hint(hints.get(&destination), Some(column)); epoch_seconds(decoded.column(positions[column]), hint) }) .transpose()?; @@ -1365,7 +1365,7 @@ fn reduce( }) } -/// `hints` apply only to frame columns, not to reductions of an earlier group. +/// `hints` describe the input columns. fn group_rows( rows: RowsView<'_>, plan: &GroupPlan, @@ -1447,6 +1447,26 @@ fn group_rows( Ok(output) } +/// Hints of a group stage's outputs: keys, and min, max and mode, which keep +/// their input's stored values. +fn group_output_hints(plan: &GroupPlan, hints: Option<&ColumnHints>) -> ColumnHints { + let mut output = ColumnHints::new(); + for key in &plan.keys { + if let Some(hint) = column_hint(hints, Some(key)) { + output.insert(key.clone(), hint.clone()); + } + } + for value in &plan.values { + match column_hint(hints, value.column.as_deref()) + .filter(|_| matches!(value.operation.as_str(), "min" | "max" | "mode")) + { + Some(hint) => output.insert(value.name.clone(), hint.clone()), + None => output.remove(&value.name), + }; + } + output +} + fn aggregate_result( frame: &MaterializedFrame, hints: Option<&ColumnHints>, @@ -1454,6 +1474,11 @@ fn aggregate_result( cutoffs: &[i64], ) -> Result { validate_reduction(&plan.operation, plan.column.as_deref())?; + let mut stage_hints = Vec::with_capacity(plan.groups.len()); + for group in &plan.groups { + let output = group_output_hints(group, stage_hints.last().or(hints)); + stage_hints.push(output); + } let mut values = Vec::with_capacity(cutoffs.len()); for (anchor, cutoff) in cutoffs.iter().copied().enumerate() { let frame_view = RowsView::Frame { @@ -1463,8 +1488,8 @@ fn aggregate_result( }; let value = if let Some((first, remaining)) = plan.groups.split_first() { let mut grouped = group_rows(frame_view, first, cutoff, hints)?; - for group in remaining { - grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, None)?; + for (group, hints) in remaining.iter().zip(&stage_hints) { + grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, Some(hints))?; } reduce( RowsView::Owned(&grouped), @@ -1472,7 +1497,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, - None, + column_hint(stage_hints.last(), plan.column.as_deref()), )? } else { reduce( @@ -1618,9 +1643,7 @@ fn count_frame( .map(|(node_id, _)| *node_id) .collect::>(); let decoded = table_rows(table.as_ref(), &flat_nodes, &projection_indices)?; - let hint = hints - .get(&destination) - .and_then(|hints| hints.get(time_column)); + let hint = column_hint(hints.get(&destination), Some(time_column)); let times = epoch_seconds(decoded.column(0), hint)?; let mut row = 0; for (anchor, rows) in rows_by_anchor.iter().enumerate() { From 46067f798254ba3ef0cd7698b2b421791c44df81 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Wed, 4 Nov 2026 22:24:08 +0000 Subject: [PATCH 04/10] Test sub-second rounding and grouped recency for integer times Co-Authored-By: Claude Opus 5.5 --- .../tests/test_temporal_int_time_units.py | 160 +++++++++++++++--- 1 file changed, 132 insertions(+), 28 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py index 2f13fc8f4..109e125f7 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py @@ -19,6 +19,15 @@ EXPECTED_IDS = [4, 3, 2] +def encode_seconds(value, hint): + if "timestamp_format" in hint: + return int( + datetime.strptime(str(value), "%Y%m%d").replace(tzinfo=UTC).timestamp() + ) + scale = {"seconds": 1, "millis": 10**3, "micros": 10**6, "nanos": 10**9} + return value // scale[hint["timestamp_unit"]] + + def encode(seconds, hint): if "timestamp_format" in hint: return int(datetime.fromtimestamp(seconds, UTC).strftime("%Y%m%d")) @@ -26,28 +35,17 @@ def encode(seconds, hint): return seconds * scale[hint["timestamp_unit"]] -@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) -@pytest.mark.parametrize( - "hint", - [ - {"timestamp_unit": "seconds"}, - {"timestamp_unit": "millis"}, - {"timestamp_unit": "micros"}, - {"timestamp_unit": "nanos"}, - {"timestamp_format": "%Y%m%d"}, - ], - ids=["s", "ms", "us", "ns", "yyyymmdd"], -) -def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): - times = [CUTOFF - offset for offset in OFFSETS] +def open_engine(tmp_path, event_times, hint, sort_key): + """`event_times` are raw `event_time` values; `time` is their floored second.""" + seconds = [encode_seconds(value, hint) for value in event_times] pq.write_table(pa.table({"id": [1]}), tmp_path / "users.parquet") pq.write_table( pa.table( { - "id": list(range(1, len(times) + 1)), - "owner": [1] * len(times), - "time": pa.array(times, type=pa.timestamp("s")), - "event_time": pa.array([encode(t, hint) for t in times], pa.int64()), + "id": list(range(1, len(event_times) + 1)), + "owner": [1] * len(event_times), + "time": pa.array(seconds, type=pa.timestamp("s")), + "event_time": pa.array(event_times, pa.int64()), } ), tmp_path / "events.parquet", @@ -111,6 +109,10 @@ def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): for relation in json.loads(engine.get_type_mapping())["edge_relations"] if relation["src_type"] == "users" ) + return engine, relation + + +def run(engine, relation, aggregates=()): plan = json.dumps( { "frames": [ @@ -122,7 +124,7 @@ def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): "start_seconds": -3 * DAY, "stop_seconds": 0, "distinct": True, - "columns": ["id"], + "columns": ["id", "owner"], } ], "sequences": [ @@ -134,21 +136,35 @@ def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): "limit": None, } ], - "aggregates": [ - { - "name": "r", - "frame": "f", - "operation": "recency", - "column": "event_time", - } - ], + "aggregates": [{"frame": "f", **aggregate} for aggregate in aggregates], } ) roots = {"users": engine.lookup_node_ids("users", pa.array([1], pa.int64()))} cutoffs = {"users": [CUTOFF]} - result = engine.query_temporal(roots, cutoffs, plan, output_format="inspectable") lengths = engine.query_temporal_lengths(roots, cutoffs, plan) + return result, lengths + + +@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) +@pytest.mark.parametrize( + "hint", + [ + {"timestamp_unit": "seconds"}, + {"timestamp_unit": "millis"}, + {"timestamp_unit": "micros"}, + {"timestamp_unit": "nanos"}, + {"timestamp_format": "%Y%m%d"}, + ], + ids=["s", "ms", "us", "ns", "yyyymmdd"], +) +def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key + ) + recency = {"name": "r", "operation": "recency", "column": "event_time"} + result, lengths = run(engine, relation, [recency]) sequence = result["sequences"][0] assert sequence["batch"].column("id").to_pylist() == EXPECTED_IDS @@ -157,3 +173,91 @@ def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): path = "sort_key_timestamps" if sort_key else "rowpack_timestamps" assert lengths["stats"][path] == len(times) assert list(result["aggregates"][0]["values"]) == [0.0] + + +@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) +def test_sub_second_times_round_up_like_ingestion(tmp_path, sort_key): + lower = (CUTOFF - 3 * DAY) * 1000 + upper = CUTOFF * 1000 + event_times = [lower - 400, lower, lower + 400, upper - 400, upper + 400] + engine, relation = open_engine( + tmp_path, event_times, {"timestamp_unit": "millis"}, sort_key + ) + result, lengths = run(engine, relation) + + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == [4, 3] + assert list(sequence["timestamps"]) == [CUTOFF, CUTOFF - 3 * DAY + 1] + assert lengths["sequences"][0]["lengths"].to_pylist() == [2] + path = "sort_key_timestamps" if sort_key else "rowpack_timestamps" + assert lengths["stats"][path] == len(event_times) + + +def test_grouped_recency_uses_declared_unit(tmp_path): + hint = {"timestamp_unit": "millis"} + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key=False + ) + count = {"name": "n", "operation": "count", "column": None} + latest = {"name": "m", "operation": "max", "column": "event_time"} + result, _ = run( + engine, + relation, + [ + { + "name": "by_time", + "groups": [{"keys": ["event_time"], "values": [count]}], + "operation": "recency", + "column": "event_time", + }, + { + "name": "of_max", + "groups": [{"keys": ["owner"], "values": [latest]}], + "operation": "recency", + "column": "m", + }, + { + "name": "grouped", + "groups": [ + { + "keys": ["owner"], + "values": [{**latest, "name": "r", "operation": "recency"}], + } + ], + "operation": "max", + "column": "r", + }, + { + "name": "two_stages", + "groups": [ + {"keys": ["id", "owner"], "values": [latest]}, + {"keys": ["owner"], "values": [{**latest, "column": "m"}]}, + ], + "operation": "recency", + "column": "m", + }, + { + "name": "overwritten_key", + "groups": [ + { + "keys": ["event_time"], + "values": [{**count, "name": "event_time"}], + } + ], + "operation": "recency", + "column": "event_time", + }, + {"name": "max", "operation": "max", "column": "event_time"}, + ], + ) + + values = {item["name"]: list(item["values"]) for item in result["aggregates"]} + assert values == { + "by_time": [0.0], + "of_max": [0.0], + "grouped": [0.0], + "two_stages": [0.0], + "overwritten_key": [float(CUTOFF - 1)], + "max": [encode(CUTOFF, hint)], + } From 459c6c1c7dca3df4024e36d7a75d541a38c3af13 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Wed, 4 Nov 2026 22:58:13 +0000 Subject: [PATCH 05/10] Fail temporal queries when the persisted graph config is unreadable Falling back to epoch seconds would silently return wrong rows for hinted integer time columns. Only hinted columns are parsed strictly, so config fields from another version fail nothing else, and sampling still opens the index. Co-Authored-By: Claude Opus 5.5 --- packages/diskgraph/diskgraph-pybind/README.md | 6 +- .../tests/test_temporal_int_time_units.py | 44 ++++++- .../diskgraph/diskgraph-pybind/src/lib.rs | 7 +- .../diskgraph-pybind/src/temporal_query.rs | 108 +++++++++++------- 4 files changed, 111 insertions(+), 54 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/README.md b/packages/diskgraph/diskgraph-pybind/README.md index b9218398c..55dc68d31 100644 --- a/packages/diskgraph/diskgraph-pybind/README.md +++ b/packages/diskgraph/diskgraph-pybind/README.md @@ -266,9 +266,9 @@ Windows are half-open relative to each aligned cutoff: bounds require `time_column`. An integer time column is read in the `timestamp_unit` or `timestamp_format` it was ingested with. `recency` reads such a column, or a group key or `min`/`max`/`mode` of it, the same way; -other reductions and projections return the stored values. Selected rows are -sorted by timestamp and then primary key before a sequence optionally reverses -and truncates them. +other reductions and projections return the stored values. An unreadable +`diskgraph_config.json` fails the query. Selected rows are sorted by timestamp +and then primary key before a sequence optionally reverses and truncates them. `distinct` deduplicates destination dense IDs per root. Available reductions are `count`, `count_distinct`, `sum`, `mean`, `min`, diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py index 109e125f7..34b077496 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py @@ -35,7 +35,7 @@ def encode(seconds, hint): return seconds * scale[hint["timestamp_unit"]] -def open_engine(tmp_path, event_times, hint, sort_key): +def open_engine(tmp_path, event_times, hint, sort_key, dtype="int64"): """`event_times` are raw `event_time` values; `time` is their floored second.""" seconds = [encode_seconds(value, hint) for value in event_times] pq.write_table(pa.table({"id": [1]}), tmp_path / "users.parquet") @@ -45,7 +45,9 @@ def open_engine(tmp_path, event_times, hint, sort_key): "id": list(range(1, len(event_times) + 1)), "owner": [1] * len(event_times), "time": pa.array(seconds, type=pa.timestamp("s")), - "event_time": pa.array(event_times, pa.int64()), + "event_time": pa.array( + event_times, {"int64": pa.int64(), "u_int64": pa.uint64()}[dtype] + ), } ), tmp_path / "events.parquet", @@ -71,7 +73,7 @@ def open_engine(tmp_path, event_times, hint, sort_key): {"name": "id", "ingested_dtype": "int64"}, {"name": "owner", "ingested_dtype": "int64"}, {"name": "time", "ingested_dtype": "timestamp"}, - {"name": "event_time", "ingested_dtype": "int64", **hint}, + {"name": "event_time", "ingested_dtype": dtype, **hint}, ], "fkeys": [ { @@ -146,6 +148,7 @@ def run(engine, relation, aggregates=()): return result, lengths +@pytest.mark.parametrize("dtype", ["int64", "u_int64"]) @pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) @pytest.mark.parametrize( "hint", @@ -158,10 +161,10 @@ def run(engine, relation, aggregates=()): ], ids=["s", "ms", "us", "ns", "yyyymmdd"], ) -def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key): +def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key, dtype): times = [CUTOFF - offset for offset in OFFSETS] engine, relation = open_engine( - tmp_path, [encode(t, hint) for t in times], hint, sort_key + tmp_path, [encode(t, hint) for t in times], hint, sort_key, dtype ) recency = {"name": "r", "operation": "recency", "column": "event_time"} result, lengths = run(engine, relation, [recency]) @@ -261,3 +264,34 @@ def test_grouped_recency_uses_declared_unit(tmp_path): "overwritten_key": [float(CUTOFF - 1)], "max": [encode(CUTOFF, hint)], } + + +def test_unreadable_config_fails_temporal_queries(tmp_path): + engine, relation = open_engine( + tmp_path, [CUTOFF], {"timestamp_unit": "seconds"}, sort_key=True + ) + (tmp_path / "index" / "diskgraph_config.json").write_text("{") + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + + with pytest.raises(diskgraph.DiskgraphConfigError, match="diskgraph_config.json"): + run(engine, relation) + + +def test_unknown_config_fields_outside_time_columns_are_ignored(tmp_path): + hint = {"timestamp_unit": "millis"} + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key=False + ) + path = tmp_path / "index" / "diskgraph_config.json" + config = json.loads(path.read_text()) + config["future_option"] = 1 + columns = config["tables"][1]["columns"] + columns[0]["ingested_dtype"] = "future_dtype" + columns[2].update(timestamp_format="%Y", semantic_type="future_stype") + columns[3]["semantic_type"] = "future_stype" + path.write_text(json.dumps(config)) + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + + result, _ = run(engine, relation) + assert result["sequences"][0]["batch"].column("id").to_pylist() == EXPECTED_IDS diff --git a/packages/diskgraph/diskgraph-pybind/src/lib.rs b/packages/diskgraph/diskgraph-pybind/src/lib.rs index 7ce91e727..29b633351 100644 --- a/packages/diskgraph/diskgraph-pybind/src/lib.rs +++ b/packages/diskgraph/diskgraph-pybind/src/lib.rs @@ -493,7 +493,8 @@ impl OutputHandler for PyOutputHandler { #[pyclass] pub struct SamplerEngine { reader: Arc, - time_hints: Arc, + /// A config error is reported by temporal queries, not by sampling. + time_hints: Arc>, engine: Arc, pool: StatePool, sample_count: Arc, @@ -773,7 +774,7 @@ impl SamplerEngine { .detach(move || { temporal_query::execute( reader.as_ref(), - time_hints.as_ref(), + time_hints.as_ref().as_ref().map_err(Clone::clone)?, &root_type, &roots, &cutoffs, @@ -820,7 +821,7 @@ impl SamplerEngine { .detach(move || { temporal_query::execute_lengths( reader.as_ref(), - time_hints.as_ref(), + time_hints.as_ref().as_ref().map_err(Clone::clone)?, &root_type, &roots, &cutoffs, diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index b5cc121fe..d121e372c 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -27,9 +27,7 @@ use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; use diskgraph_engine::edge_unpacker::EdgeUnpacker; use diskgraph_index::graph::{GraphIndexReader, TableReader}; -use diskgraph_ingest::graph_config::{ - read_persisted_config, IngestedType, DISKGRAPH_CONFIG_FILENAME, -}; +use diskgraph_ingest::graph_config::{ColumnSpec, IngestedType, DISKGRAPH_CONFIG_FILENAME}; use diskgraph_rowpack::{RowAccess, RowAccessHints, RowAccessStats}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; @@ -553,49 +551,73 @@ type ColumnHints = HashMap; pub(crate) type TimeHints = HashMap; /// Reads [`TimeHints`] from the index's persisted graph config. Indexes without -/// one read integer times as epoch seconds. -pub(crate) fn time_hints(root: &Path) -> TimeHints { - if !root.join(DISKGRAPH_CONFIG_FILENAME).exists() { - return TimeHints::new(); - } - let config = match read_persisted_config(root) { - Ok(config) => config, - Err(error) => { - tracing::warn!("temporal queries read integer times as epoch seconds: {error:#}"); - return TimeHints::new(); - } +/// one read integer times as epoch seconds; an unreadable one is an error. Only +/// the time fields of integer columns are parsed strictly, so config fields this +/// version does not know fail nothing else. +pub(crate) fn time_hints(root: &Path) -> Result { + #[derive(Deserialize)] + struct Config { + tables: Vec, + } + #[derive(Deserialize)] + struct Table { + name: String, + columns: Vec>, + } + let path = root.join(DISKGRAPH_CONFIG_FILENAME); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(TimeHints::new()), + Err(error) => return Err(format!("read {}: {error}", path.display())), }; - config - .tables - .iter() - .map(|table| { - let hints = table - .columns - .iter() - .filter(|column| { - (column.timestamp_unit.is_some() || column.timestamp_format.is_some()) - && matches!( - column.ingested_dtype, - IngestedType::Int8 - | IngestedType::Int16 - | IngestedType::Int32 - | IngestedType::Int64 - | IngestedType::UInt8 - | IngestedType::UInt16 - | IngestedType::UInt32 - | IngestedType::UInt64 - ) - }) - .map(|column| { - ( - column.name.clone(), - crate::bp_col_stats::row_access_hints(column), + let config: Config = serde_json::from_slice(&bytes) + .map_err(|error| format!("parse {}: {error}", path.display()))?; + let mut hints = TimeHints::new(); + for table in config.tables { + let mut columns = ColumnHints::new(); + for column in table.columns { + let Some(dtype) = column + .get("ingested_dtype") + .or_else(|| column.get("dtype")) + .filter(|dtype| { + matches!( + serde_json::from_value((*dtype).clone()), + Ok(IngestedType::Int8 + | IngestedType::Int16 + | IngestedType::Int32 + | IngestedType::Int64 + | IngestedType::UInt8 + | IngestedType::UInt16 + | IngestedType::UInt32 + | IngestedType::UInt64) ) }) - .collect(); - (table.name.clone(), hints) - }) - .collect() + else { + continue; + }; + let mut fields = serde_json::Map::from_iter([("ingested_dtype".into(), dtype.clone())]); + for key in ["name", "timestamp_unit", "timestamp_format"] { + if let Some(value) = column.get(key) { + fields.insert(key.into(), value.clone()); + } + } + let column: ColumnSpec = serde_json::from_value(serde_json::Value::Object(fields)) + .map_err(|error| { + format!( + "parse {}: table {:?} column {}: {error}", + path.display(), + table.name, + column.get("name").unwrap_or(&serde_json::Value::Null) + ) + })?; + if column.timestamp_unit.is_some() || column.timestamp_format.is_some() { + let hint = crate::bp_col_stats::row_access_hints(&column); + columns.insert(column.name, hint); + } + } + hints.insert(table.name, columns); + } + Ok(hints) } /// Epoch seconds of each value. A hinted integer column decodes exactly as From d291634e63d36a342c9e278296565a4798e0c020 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 5 Nov 2026 18:09:01 +0000 Subject: [PATCH 06/10] Build each recency's time reader once per aggregate A recency over a hinted integer column built a new reader for every anchor and every group. Build one per reduction when the aggregate is planned and reuse it. Co-Authored-By: Claude Opus 5.5 --- .../diskgraph-pybind/src/temporal_query.rs | 92 +++++++++++++------ 1 file changed, 65 insertions(+), 27 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index d121e372c..5a8e86d4f 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -620,25 +620,37 @@ pub(crate) fn time_hints(root: &Path) -> Result { Ok(hints) } -/// Epoch seconds of each value. A hinted integer column decodes exactly as -/// ingestion and its edge sort keys read it. +/// Reads a hinted integer time column exactly as ingestion and its edge sort +/// keys read it. +struct TimeReader(RowAccess<'static>); + +impl TimeReader { + fn new(hint: &RowAccessHints) -> Self { + Self(RowAccess::new(vec![hint.clone()], RowAccessStats::new(None))) + } + + fn epoch_seconds(&mut self, array: &ArrayRef) -> Result>, String> { + self.0.set_cols(std::slice::from_ref(array)); + (0..array.len()) + .map(|row| { + self.0.set_row(row); + self.0.get_timestamp(0).map_err(|error| error.to_string()) + }) + .collect() + } +} + +/// Epoch seconds of each value. fn epoch_seconds( array: &ArrayRef, hint: Option<&RowAccessHints>, ) -> Result>, String> { - let Some(hint) = hint else { - return (0..array.len()) + match hint { + Some(hint) => TimeReader::new(hint).epoch_seconds(array), + None => (0..array.len()) .map(|row| scalar_at(array.as_ref(), row)?.epoch_seconds()) - .collect(); - }; - let mut access = RowAccess::new(vec![hint.clone()], RowAccessStats::new(None)); - access.set_cols(std::slice::from_ref(array)); - (0..array.len()) - .map(|row| { - access.set_row(row); - access.get_timestamp(0).map_err(|error| error.to_string()) - }) - .collect() + .collect(), + } } fn relation_columns( @@ -1252,7 +1264,7 @@ fn reduce( column: Option<&str>, cutoff: i64, sum_type: Option, - hint: Option<&RowAccessHints>, + time: Option<&mut TimeReader>, ) -> Result { validate_reduction(operation, column)?; if sum_type.is_some() && operation != "sum" { @@ -1316,8 +1328,8 @@ fn reduce( .iter() .map(ScalarValue::epoch_seconds) .collect::, _>>()?; - if hint.is_some() { - seconds = epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef), hint)?; + if let Some(time) = time { + seconds = time.epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef))?; } let latest = seconds .into_iter() @@ -1387,12 +1399,12 @@ fn reduce( }) } -/// `hints` describe the input columns. +/// `times[i]` reads the time column of `plan.values[i]`. fn group_rows( rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64, - hints: Option<&ColumnHints>, + times: &mut [Option], ) -> Result, String> { if plan.keys.is_empty() || plan.values.is_empty() { return Err("group_by requires keys and values".into()); @@ -1460,7 +1472,7 @@ fn group_rows( reduction.column.as_deref(), cutoff, reduction.sum_type, - column_hint(hints, reduction.column.as_deref()), + times[reduction_index].as_mut(), )?, ); } @@ -1489,6 +1501,17 @@ fn group_output_hints(plan: &GroupPlan, hints: Option<&ColumnHints>) -> ColumnHi output } +/// Reader of a `recency`'s hinted time column. +fn time_reader( + hints: Option<&ColumnHints>, + operation: &str, + column: Option<&str>, +) -> Option { + column_hint(hints, column) + .filter(|_| operation == "recency") + .map(TimeReader::new) +} + fn aggregate_result( frame: &MaterializedFrame, hints: Option<&ColumnHints>, @@ -1497,10 +1520,24 @@ fn aggregate_result( ) -> Result { validate_reduction(&plan.operation, plan.column.as_deref())?; let mut stage_hints = Vec::with_capacity(plan.groups.len()); + let mut stage_times = Vec::with_capacity(plan.groups.len()); for group in &plan.groups { - let output = group_output_hints(group, stage_hints.last().or(hints)); + let input = stage_hints.last().or(hints); + stage_times.push( + group + .values + .iter() + .map(|value| time_reader(input, &value.operation, value.column.as_deref())) + .collect::>(), + ); + let output = group_output_hints(group, input); stage_hints.push(output); } + let mut time = time_reader( + stage_hints.last().or(hints), + &plan.operation, + plan.column.as_deref(), + ); let mut values = Vec::with_capacity(cutoffs.len()); for (anchor, cutoff) in cutoffs.iter().copied().enumerate() { let frame_view = RowsView::Frame { @@ -1508,10 +1545,11 @@ fn aggregate_result( start: frame.offsets[anchor], stop: frame.offsets[anchor + 1], }; - let value = if let Some((first, remaining)) = plan.groups.split_first() { - let mut grouped = group_rows(frame_view, first, cutoff, hints)?; - for (group, hints) in remaining.iter().zip(&stage_hints) { - grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, Some(hints))?; + let mut stages = plan.groups.iter().zip(&mut stage_times); + let value = if let Some((first, times)) = stages.next() { + let mut grouped = group_rows(frame_view, first, cutoff, times)?; + for (group, times) in stages { + grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, times)?; } reduce( RowsView::Owned(&grouped), @@ -1519,7 +1557,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, - column_hint(stage_hints.last(), plan.column.as_deref()), + time.as_mut(), )? } else { reduce( @@ -1528,7 +1566,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, - column_hint(hints, plan.column.as_deref()), + time.as_mut(), )? }; values.push(value); From f3eab7fcc67c1aabbabb1a137c1f655fbe5a35c9 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 5 Nov 2026 18:15:24 +0000 Subject: [PATCH 07/10] Read time hints in the reader's single graph config pass SamplerEngine parsed diskgraph_config.json a second time at open for temporal time hints. The index reader already parses it for semantic types, so it now also keeps the columns that declare a timestamp_unit or timestamp_format, and an unreadable config still fails temporal queries only. Co-Authored-By: Claude Opus 5.5 --- .../diskgraph/diskgraph-index/src/graph.rs | 56 ++++++++++++++++--- .../diskgraph/diskgraph-pybind/src/lib.rs | 6 +- .../diskgraph-pybind/src/temporal_query.rs | 39 +++---------- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 64aeff626..dc900aaed 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -964,6 +964,9 @@ impl<'a> crate::builder::ProgressSink for ShardProgressAdapter<'a> { } } +/// Persisted graph config column objects, by table. +pub type TimeColumns = HashMap>>; + pub struct GraphIndexReader { metadata: DatasetMetadata, tables: HashMap>, @@ -972,6 +975,10 @@ pub struct GraphIndexReader { /// follow-up). Used by TT-gen resolver to dispatch per-token observation /// for multicategorical sources without per-query plan-JSON plumbing. column_semantic_types: HashMap<(String, String), String>, + /// Columns of `diskgraph_config.json` that declare a `timestamp_unit` or + /// `timestamp_format`. Empty when the file is absent, an error when it is + /// unreadable. + time_columns: Result, /// Unified view of edge relations across parent + every loaded extension. /// Extension relations have their `src_type` rewritten to the namespaced /// `/` form so a consumer (e.g. the sampler engine) @@ -1126,7 +1133,7 @@ impl GraphIndexReader { )?; tables.insert(node_type.name.clone(), Arc::new(table)); } - let column_semantic_types = load_column_semantic_types(root); + let (column_semantic_types, time_columns) = load_graph_config(root); let merged_edge_relations = metadata.edge_relations.clone(); tracing::info!( tables = tables.len(), @@ -1137,6 +1144,7 @@ impl GraphIndexReader { metadata, tables, column_semantic_types, + time_columns, merged_edge_relations, // `open` / `open_with_external_node_types` produce a reader over // a single index. `open_with_tt` overwrites this after @@ -1191,6 +1199,12 @@ impl GraphIndexReader { .map(String::as_str) } + /// Columns of the persisted graph config that declare a `timestamp_unit` + /// or `timestamp_format`, by table. + pub fn time_columns(&self) -> Result<&TimeColumns, &str> { + self.time_columns.as_ref().map_err(String::as_str) + } + /// Open the parent DG index together with exactly one seed-table /// extension. A bare `tt_ref` keeps legacy behavior and opens /// `/train_tables//`; explicit @@ -1370,14 +1384,21 @@ fn merge_extension_into( Ok(()) } -/// Best-effort load of `{root}/diskgraph_config.json`. Returns an empty map -/// when the file is missing or malformed; the resolver treats absence as -/// "stype unknown" and falls back to the legacy whole-cell observe path. -fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> { +/// Loads `{root}/diskgraph_config.json` once. Semantic types are best-effort: +/// an empty map when the file is missing or malformed; the resolver treats +/// absence as "stype unknown" and falls back to the legacy whole-cell observe +/// path. Time columns are empty when the file is missing and an error when it +/// is unreadable. +fn load_graph_config( + root: &Path, +) -> (HashMap<(String, String), String>, Result) { let path = root.join("diskgraph_config.json"); let bytes = match fs::read(&path) { Ok(b) => b, - Err(_) => return HashMap::new(), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return (HashMap::new(), Ok(TimeColumns::new())); + } + Err(err) => return (HashMap::new(), Err(format!("read {}: {err}", path.display()))), }; let value: serde_json::Value = match serde_json::from_slice(&bytes) { Ok(v) => v, @@ -1386,12 +1407,14 @@ fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> "diskgraph_config.json present but unparseable: {err}; \ multicat dispatch will fall back to whole-cell" ); - return HashMap::new(); + return (HashMap::new(), Err(format!("parse {}: {err}", path.display()))); } }; + let time_columns = time_columns(&value) + .ok_or_else(|| format!("parse {}: unexpected tables layout", path.display())); let mut out = HashMap::new(); let Some(tables) = value.get("tables").and_then(|t| t.as_array()) else { - return out; + return (out, time_columns); }; for table in tables { let Some(table_name) = table.get("name").and_then(|n| n.as_str()) else { @@ -1413,7 +1436,22 @@ fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> ); } } - out + (out, time_columns) +} + +fn time_columns(config: &serde_json::Value) -> Option { + let mut out = TimeColumns::new(); + for table in config.get("tables")?.as_array()? { + let mut columns = Vec::new(); + for column in table.get("columns")?.as_array()? { + let column = column.as_object()?; + if column.contains_key("timestamp_unit") || column.contains_key("timestamp_format") { + columns.push(column.clone()); + } + } + out.insert(table.get("name")?.as_str()?.to_string(), columns); + } + Some(out) } pub struct TableReader { diff --git a/packages/diskgraph/diskgraph-pybind/src/lib.rs b/packages/diskgraph/diskgraph-pybind/src/lib.rs index 29b633351..6c4b46c70 100644 --- a/packages/diskgraph/diskgraph-pybind/src/lib.rs +++ b/packages/diskgraph/diskgraph-pybind/src/lib.rs @@ -693,10 +693,14 @@ impl SamplerEngine { ); let engine = DiskGraphEngine::from_reader(Arc::clone(&reader), random_seed, num_threads) .map_err(map_engine_err)?; + let time_hints = reader + .time_columns() + .map_err(str::to_string) + .and_then(temporal_query::time_hints); info!("SamplerEngine initialized successfully"); Ok(Self { reader, - time_hints: Arc::new(temporal_query::time_hints(root)), + time_hints: Arc::new(time_hints), engine: Arc::new(engine), pool: StatePool::new(), sample_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index 5a8e86d4f..219ba5791 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -11,7 +11,6 @@ use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::path::Path; use std::sync::{Arc, LazyLock}; use std::time::Instant; @@ -26,7 +25,7 @@ use arrow_array::RecordBatch; use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; use diskgraph_engine::edge_unpacker::EdgeUnpacker; -use diskgraph_index::graph::{GraphIndexReader, TableReader}; +use diskgraph_index::graph::{GraphIndexReader, TableReader, TimeColumns}; use diskgraph_ingest::graph_config::{ColumnSpec, IngestedType, DISKGRAPH_CONFIG_FILENAME}; use diskgraph_rowpack::{RowAccess, RowAccessHints, RowAccessStats}; use pyo3::prelude::*; @@ -550,32 +549,14 @@ type ColumnHints = HashMap; /// Ingest hints of integer columns declared with a time unit or format, by table. pub(crate) type TimeHints = HashMap; -/// Reads [`TimeHints`] from the index's persisted graph config. Indexes without -/// one read integer times as epoch seconds; an unreadable one is an error. Only -/// the time fields of integer columns are parsed strictly, so config fields this -/// version does not know fail nothing else. -pub(crate) fn time_hints(root: &Path) -> Result { - #[derive(Deserialize)] - struct Config { - tables: Vec
, - } - #[derive(Deserialize)] - struct Table { - name: String, - columns: Vec>, - } - let path = root.join(DISKGRAPH_CONFIG_FILENAME); - let bytes = match std::fs::read(&path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(TimeHints::new()), - Err(error) => return Err(format!("read {}: {error}", path.display())), - }; - let config: Config = serde_json::from_slice(&bytes) - .map_err(|error| format!("parse {}: {error}", path.display()))?; +/// [`TimeHints`] of the time columns the index's persisted graph config +/// declares. Only the time fields of integer columns are parsed strictly, so +/// config fields this version does not know fail nothing else. +pub(crate) fn time_hints(tables: &TimeColumns) -> Result { let mut hints = TimeHints::new(); - for table in config.tables { + for (table, table_columns) in tables { let mut columns = ColumnHints::new(); - for column in table.columns { + for column in table_columns { let Some(dtype) = column .get("ingested_dtype") .or_else(|| column.get("dtype")) @@ -604,9 +585,7 @@ pub(crate) fn time_hints(root: &Path) -> Result { let column: ColumnSpec = serde_json::from_value(serde_json::Value::Object(fields)) .map_err(|error| { format!( - "parse {}: table {:?} column {}: {error}", - path.display(), - table.name, + "parse {DISKGRAPH_CONFIG_FILENAME}: table {table:?} column {}: {error}", column.get("name").unwrap_or(&serde_json::Value::Null) ) })?; @@ -615,7 +594,7 @@ pub(crate) fn time_hints(root: &Path) -> Result { columns.insert(column.name, hint); } } - hints.insert(table.name, columns); + hints.insert(table.clone(), columns); } Ok(hints) } From cdb5d6e710cc138c1c5059efb6629f3155fe28fc Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 5 Nov 2026 18:22:27 +0000 Subject: [PATCH 08/10] Keep persisted time columns in file order Duplicate tables and the first reported config error then follow the file, as before the single parse. Co-Authored-By: Claude Opus 5.5 --- packages/diskgraph/diskgraph-index/src/graph.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index dc900aaed..726a7be1a 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -964,8 +964,9 @@ impl<'a> crate::builder::ProgressSink for ShardProgressAdapter<'a> { } } -/// Persisted graph config column objects, by table. -pub type TimeColumns = HashMap>>; +/// Raw graph config objects of the columns that declare a `timestamp_unit` or +/// `timestamp_format`, by table in file order. +pub type TimeColumns = Vec<(String, Vec>)>; pub struct GraphIndexReader { metadata: DatasetMetadata, @@ -1410,7 +1411,7 @@ fn load_graph_config( return (HashMap::new(), Err(format!("parse {}: {err}", path.display()))); } }; - let time_columns = time_columns(&value) + let time_columns = parse_time_columns(&value) .ok_or_else(|| format!("parse {}: unexpected tables layout", path.display())); let mut out = HashMap::new(); let Some(tables) = value.get("tables").and_then(|t| t.as_array()) else { @@ -1439,7 +1440,8 @@ fn load_graph_config( (out, time_columns) } -fn time_columns(config: &serde_json::Value) -> Option { +/// Strict about the layout, unlike the semantic types walk. +fn parse_time_columns(config: &serde_json::Value) -> Option { let mut out = TimeColumns::new(); for table in config.get("tables")?.as_array()? { let mut columns = Vec::new(); @@ -1449,7 +1451,7 @@ fn time_columns(config: &serde_json::Value) -> Option { columns.push(column.clone()); } } - out.insert(table.get("name")?.as_str()?.to_string(), columns); + out.push((table.get("name")?.as_str()?.to_string(), columns)); } Some(out) } From 4b713cd415ec531f4955b3fbdd943e1da79b9111 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 5 Nov 2026 18:53:35 +0000 Subject: [PATCH 09/10] Revert "Build each recency's time reader once per aggregate" This reverts commit d291634e6. It gave no measurable gain. Co-Authored-By: Claude Opus 5.5 --- .../diskgraph-pybind/src/temporal_query.rs | 92 ++++++------------- 1 file changed, 27 insertions(+), 65 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index 219ba5791..bdcc9e1d0 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -599,37 +599,25 @@ pub(crate) fn time_hints(tables: &TimeColumns) -> Result { Ok(hints) } -/// Reads a hinted integer time column exactly as ingestion and its edge sort -/// keys read it. -struct TimeReader(RowAccess<'static>); - -impl TimeReader { - fn new(hint: &RowAccessHints) -> Self { - Self(RowAccess::new(vec![hint.clone()], RowAccessStats::new(None))) - } - - fn epoch_seconds(&mut self, array: &ArrayRef) -> Result>, String> { - self.0.set_cols(std::slice::from_ref(array)); - (0..array.len()) - .map(|row| { - self.0.set_row(row); - self.0.get_timestamp(0).map_err(|error| error.to_string()) - }) - .collect() - } -} - -/// Epoch seconds of each value. +/// Epoch seconds of each value. A hinted integer column decodes exactly as +/// ingestion and its edge sort keys read it. fn epoch_seconds( array: &ArrayRef, hint: Option<&RowAccessHints>, ) -> Result>, String> { - match hint { - Some(hint) => TimeReader::new(hint).epoch_seconds(array), - None => (0..array.len()) + let Some(hint) = hint else { + return (0..array.len()) .map(|row| scalar_at(array.as_ref(), row)?.epoch_seconds()) - .collect(), - } + .collect(); + }; + let mut access = RowAccess::new(vec![hint.clone()], RowAccessStats::new(None)); + access.set_cols(std::slice::from_ref(array)); + (0..array.len()) + .map(|row| { + access.set_row(row); + access.get_timestamp(0).map_err(|error| error.to_string()) + }) + .collect() } fn relation_columns( @@ -1243,7 +1231,7 @@ fn reduce( column: Option<&str>, cutoff: i64, sum_type: Option, - time: Option<&mut TimeReader>, + hint: Option<&RowAccessHints>, ) -> Result { validate_reduction(operation, column)?; if sum_type.is_some() && operation != "sum" { @@ -1307,8 +1295,8 @@ fn reduce( .iter() .map(ScalarValue::epoch_seconds) .collect::, _>>()?; - if let Some(time) = time { - seconds = time.epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef))?; + if hint.is_some() { + seconds = epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef), hint)?; } let latest = seconds .into_iter() @@ -1378,12 +1366,12 @@ fn reduce( }) } -/// `times[i]` reads the time column of `plan.values[i]`. +/// `hints` describe the input columns. fn group_rows( rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64, - times: &mut [Option], + hints: Option<&ColumnHints>, ) -> Result, String> { if plan.keys.is_empty() || plan.values.is_empty() { return Err("group_by requires keys and values".into()); @@ -1451,7 +1439,7 @@ fn group_rows( reduction.column.as_deref(), cutoff, reduction.sum_type, - times[reduction_index].as_mut(), + column_hint(hints, reduction.column.as_deref()), )?, ); } @@ -1480,17 +1468,6 @@ fn group_output_hints(plan: &GroupPlan, hints: Option<&ColumnHints>) -> ColumnHi output } -/// Reader of a `recency`'s hinted time column. -fn time_reader( - hints: Option<&ColumnHints>, - operation: &str, - column: Option<&str>, -) -> Option { - column_hint(hints, column) - .filter(|_| operation == "recency") - .map(TimeReader::new) -} - fn aggregate_result( frame: &MaterializedFrame, hints: Option<&ColumnHints>, @@ -1499,24 +1476,10 @@ fn aggregate_result( ) -> Result { validate_reduction(&plan.operation, plan.column.as_deref())?; let mut stage_hints = Vec::with_capacity(plan.groups.len()); - let mut stage_times = Vec::with_capacity(plan.groups.len()); for group in &plan.groups { - let input = stage_hints.last().or(hints); - stage_times.push( - group - .values - .iter() - .map(|value| time_reader(input, &value.operation, value.column.as_deref())) - .collect::>(), - ); - let output = group_output_hints(group, input); + let output = group_output_hints(group, stage_hints.last().or(hints)); stage_hints.push(output); } - let mut time = time_reader( - stage_hints.last().or(hints), - &plan.operation, - plan.column.as_deref(), - ); let mut values = Vec::with_capacity(cutoffs.len()); for (anchor, cutoff) in cutoffs.iter().copied().enumerate() { let frame_view = RowsView::Frame { @@ -1524,11 +1487,10 @@ fn aggregate_result( start: frame.offsets[anchor], stop: frame.offsets[anchor + 1], }; - let mut stages = plan.groups.iter().zip(&mut stage_times); - let value = if let Some((first, times)) = stages.next() { - let mut grouped = group_rows(frame_view, first, cutoff, times)?; - for (group, times) in stages { - grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, times)?; + let value = if let Some((first, remaining)) = plan.groups.split_first() { + let mut grouped = group_rows(frame_view, first, cutoff, hints)?; + for (group, hints) in remaining.iter().zip(&stage_hints) { + grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, Some(hints))?; } reduce( RowsView::Owned(&grouped), @@ -1536,7 +1498,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, - time.as_mut(), + column_hint(stage_hints.last(), plan.column.as_deref()), )? } else { reduce( @@ -1545,7 +1507,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, - time.as_mut(), + column_hint(hints, plan.column.as_deref()), )? }; values.push(value); From 5ce51c6a10fa547d5bfbf0a92b89397c7918d9a4 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 5 Nov 2026 19:06:23 +0000 Subject: [PATCH 10/10] Read declared-seconds integer times as undeclared ones On the seconds clock a timestamp_unit of seconds reads every value unchanged, yet it sent recency through the RowAccess decoder per anchor: 115 ms against 63 ms on main for 4,096 anchors. Keep hints only for a format or a finer unit. Co-Authored-By: Claude Opus 5.5 --- .../diskgraph-pybind/src/temporal_query.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index bdcc9e1d0..1edb7c664 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -26,7 +26,9 @@ use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; use diskgraph_engine::edge_unpacker::EdgeUnpacker; use diskgraph_index::graph::{GraphIndexReader, TableReader, TimeColumns}; -use diskgraph_ingest::graph_config::{ColumnSpec, IngestedType, DISKGRAPH_CONFIG_FILENAME}; +use diskgraph_ingest::graph_config::{ + ColumnSpec, IngestedType, TimestampUnit, DISKGRAPH_CONFIG_FILENAME, +}; use diskgraph_rowpack::{RowAccess, RowAccessHints, RowAccessStats}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; @@ -546,7 +548,8 @@ fn scalar_at(array: &dyn Array, row: usize) -> Result { /// Ingest hints by column name, for one table. type ColumnHints = HashMap; -/// Ingest hints of integer columns declared with a time unit or format, by table. +/// Ingest hints of integer columns declared with a time format or a unit other +/// than seconds, by table. pub(crate) type TimeHints = HashMap; /// [`TimeHints`] of the time columns the index's persisted graph config @@ -589,7 +592,11 @@ pub(crate) fn time_hints(tables: &TimeColumns) -> Result { column.get("name").unwrap_or(&serde_json::Value::Null) ) })?; - if column.timestamp_unit.is_some() || column.timestamp_format.is_some() { + if column.timestamp_format.is_some() + || column + .timestamp_unit + .is_some_and(|unit| unit != TimestampUnit::Seconds) + { let hint = crate::bp_col_stats::row_access_hints(&column); columns.insert(column.name, hint); }