diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 64aeff626..726a7be1a 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -964,6 +964,10 @@ impl<'a> crate::builder::ProgressSink for ShardProgressAdapter<'a> { } } +/// Raw graph config objects of the columns that declare a `timestamp_unit` or +/// `timestamp_format`, by table in file order. +pub type TimeColumns = Vec<(String, Vec>)>; + pub struct GraphIndexReader { metadata: DatasetMetadata, tables: HashMap>, @@ -972,6 +976,10 @@ pub struct GraphIndexReader { /// follow-up). Used by TT-gen resolver to dispatch per-token observation /// for multicategorical sources without per-query plan-JSON plumbing. column_semantic_types: HashMap<(String, String), String>, + /// Columns of `diskgraph_config.json` that declare a `timestamp_unit` or + /// `timestamp_format`. Empty when the file is absent, an error when it is + /// unreadable. + time_columns: Result, /// Unified view of edge relations across parent + every loaded extension. /// Extension relations have their `src_type` rewritten to the namespaced /// `/` form so a consumer (e.g. the sampler engine) @@ -1126,7 +1134,7 @@ impl GraphIndexReader { )?; tables.insert(node_type.name.clone(), Arc::new(table)); } - let column_semantic_types = load_column_semantic_types(root); + let (column_semantic_types, time_columns) = load_graph_config(root); let merged_edge_relations = metadata.edge_relations.clone(); tracing::info!( tables = tables.len(), @@ -1137,6 +1145,7 @@ impl GraphIndexReader { metadata, tables, column_semantic_types, + time_columns, merged_edge_relations, // `open` / `open_with_external_node_types` produce a reader over // a single index. `open_with_tt` overwrites this after @@ -1191,6 +1200,12 @@ impl GraphIndexReader { .map(String::as_str) } + /// Columns of the persisted graph config that declare a `timestamp_unit` + /// or `timestamp_format`, by table. + pub fn time_columns(&self) -> Result<&TimeColumns, &str> { + self.time_columns.as_ref().map_err(String::as_str) + } + /// Open the parent DG index together with exactly one seed-table /// extension. A bare `tt_ref` keeps legacy behavior and opens /// `/train_tables//`; explicit @@ -1370,14 +1385,21 @@ fn merge_extension_into( Ok(()) } -/// Best-effort load of `{root}/diskgraph_config.json`. Returns an empty map -/// when the file is missing or malformed; the resolver treats absence as -/// "stype unknown" and falls back to the legacy whole-cell observe path. -fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> { +/// Loads `{root}/diskgraph_config.json` once. Semantic types are best-effort: +/// an empty map when the file is missing or malformed; the resolver treats +/// absence as "stype unknown" and falls back to the legacy whole-cell observe +/// path. Time columns are empty when the file is missing and an error when it +/// is unreadable. +fn load_graph_config( + root: &Path, +) -> (HashMap<(String, String), String>, Result) { let path = root.join("diskgraph_config.json"); let bytes = match fs::read(&path) { Ok(b) => b, - Err(_) => return HashMap::new(), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return (HashMap::new(), Ok(TimeColumns::new())); + } + Err(err) => return (HashMap::new(), Err(format!("read {}: {err}", path.display()))), }; let value: serde_json::Value = match serde_json::from_slice(&bytes) { Ok(v) => v, @@ -1386,12 +1408,14 @@ fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> "diskgraph_config.json present but unparseable: {err}; \ multicat dispatch will fall back to whole-cell" ); - return HashMap::new(); + return (HashMap::new(), Err(format!("parse {}: {err}", path.display()))); } }; + let time_columns = parse_time_columns(&value) + .ok_or_else(|| format!("parse {}: unexpected tables layout", path.display())); let mut out = HashMap::new(); let Some(tables) = value.get("tables").and_then(|t| t.as_array()) else { - return out; + return (out, time_columns); }; for table in tables { let Some(table_name) = table.get("name").and_then(|n| n.as_str()) else { @@ -1413,7 +1437,23 @@ fn load_column_semantic_types(root: &Path) -> HashMap<(String, String), String> ); } } - out + (out, time_columns) +} + +/// Strict about the layout, unlike the semantic types walk. +fn parse_time_columns(config: &serde_json::Value) -> Option { + let mut out = TimeColumns::new(); + for table in config.get("tables")?.as_array()? { + let mut columns = Vec::new(); + for column in table.get("columns")?.as_array()? { + let column = column.as_object()?; + if column.contains_key("timestamp_unit") || column.contains_key("timestamp_format") { + columns.push(column.clone()); + } + } + out.push((table.get("name")?.as_str()?.to_string(), columns)); + } + Some(out) } pub struct TableReader { diff --git a/packages/diskgraph/diskgraph-pybind/README.md b/packages/diskgraph/diskgraph-pybind/README.md index e56f24e55..55dc68d31 100644 --- a/packages/diskgraph/diskgraph-pybind/README.md +++ b/packages/diskgraph/diskgraph-pybind/README.md @@ -263,8 +263,12 @@ result = engine.query_temporal( Windows are half-open relative to each aligned cutoff: `(cutoff + start_seconds, cutoff + stop_seconds]`. Either bound may be null; -bounds require `time_column`. Selected rows are sorted by timestamp and then -primary key before a sequence optionally reverses and truncates them. +bounds require `time_column`. An integer time column is read in the +`timestamp_unit` or `timestamp_format` it was ingested with. `recency` reads +such a column, or a group key or `min`/`max`/`mode` of it, the same way; +other reductions and projections return the stored values. An unreadable +`diskgraph_config.json` fails the query. Selected rows are sorted by timestamp +and then primary key before a sequence optionally reverses and truncates them. `distinct` deduplicates destination dense IDs per root. Available reductions are `count`, `count_distinct`, `sum`, `mean`, `min`, diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py new file mode 100644 index 000000000..34b077496 --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_temporal_int_time_units.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integer time columns are read in their declared unit or format.""" + +import json +from datetime import UTC, datetime + +import diskgraph +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +CUTOFF = int(datetime(2024, 3, 10, tzinfo=UTC).timestamp()) +DAY = 86_400 +# Seconds before the cutoff; the window keeps (-3 days, 0]. +OFFSETS = [5 * DAY, 2 * DAY, DAY, 0, -DAY] +EXPECTED_IDS = [4, 3, 2] + + +def encode_seconds(value, hint): + if "timestamp_format" in hint: + return int( + datetime.strptime(str(value), "%Y%m%d").replace(tzinfo=UTC).timestamp() + ) + scale = {"seconds": 1, "millis": 10**3, "micros": 10**6, "nanos": 10**9} + return value // scale[hint["timestamp_unit"]] + + +def encode(seconds, hint): + if "timestamp_format" in hint: + return int(datetime.fromtimestamp(seconds, UTC).strftime("%Y%m%d")) + scale = {"seconds": 1, "millis": 10**3, "micros": 10**6, "nanos": 10**9} + return seconds * scale[hint["timestamp_unit"]] + + +def open_engine(tmp_path, event_times, hint, sort_key, dtype="int64"): + """`event_times` are raw `event_time` values; `time` is their floored second.""" + seconds = [encode_seconds(value, hint) for value in event_times] + pq.write_table(pa.table({"id": [1]}), tmp_path / "users.parquet") + pq.write_table( + pa.table( + { + "id": list(range(1, len(event_times) + 1)), + "owner": [1] * len(event_times), + "time": pa.array(seconds, type=pa.timestamp("s")), + "event_time": pa.array( + event_times, {"int64": pa.int64(), "u_int64": pa.uint64()}[dtype] + ), + } + ), + tmp_path / "events.parquet", + ) + config = { + "dataset_name": "int_time_units", + "num_shards": 1, + "tables": [ + { + "name": "users", + "primary_key": "id", + "columns": [{"name": "id", "ingested_dtype": "int64"}], + "source": { + "type": "parquet", + "paths": [str(tmp_path / "users.parquet")], + }, + }, + { + "name": "events", + "primary_key": "id", + "create_time": "time", + "columns": [ + {"name": "id", "ingested_dtype": "int64"}, + {"name": "owner", "ingested_dtype": "int64"}, + {"name": "time", "ingested_dtype": "timestamp"}, + {"name": "event_time", "ingested_dtype": dtype, **hint}, + ], + "fkeys": [ + { + "src_column": "owner", + "dst_table": "users", + "reverse": { + "time": { + "table": "events", + "column": "event_time" if sort_key else "time", + "unit": "s", + } + }, + } + ], + "source": { + "type": "parquet", + "paths": [str(tmp_path / "events.parquet")], + }, + }, + ], + } + diskgraph.ingest( + json.dumps(config), + str(tmp_path / "index"), + col_stats=False, + context_threads=1, + encode_threads=1, + pk_map_threads=1, + edges_map_threads=1, + index_threads=1, + ) + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + relation = next( + relation["name"] + for relation in json.loads(engine.get_type_mapping())["edge_relations"] + if relation["src_type"] == "users" + ) + return engine, relation + + +def run(engine, relation, aggregates=()): + plan = json.dumps( + { + "frames": [ + { + "name": "f", + "root_type": "users", + "path": [{"relation": relation, "destination_type": "events"}], + "time_column": "event_time", + "start_seconds": -3 * DAY, + "stop_seconds": 0, + "distinct": True, + "columns": ["id", "owner"], + } + ], + "sequences": [ + { + "name": "s", + "frame": "f", + "columns": ["id"], + "order": "descending", + "limit": None, + } + ], + "aggregates": [{"frame": "f", **aggregate} for aggregate in aggregates], + } + ) + roots = {"users": engine.lookup_node_ids("users", pa.array([1], pa.int64()))} + cutoffs = {"users": [CUTOFF]} + result = engine.query_temporal(roots, cutoffs, plan, output_format="inspectable") + lengths = engine.query_temporal_lengths(roots, cutoffs, plan) + return result, lengths + + +@pytest.mark.parametrize("dtype", ["int64", "u_int64"]) +@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) +@pytest.mark.parametrize( + "hint", + [ + {"timestamp_unit": "seconds"}, + {"timestamp_unit": "millis"}, + {"timestamp_unit": "micros"}, + {"timestamp_unit": "nanos"}, + {"timestamp_format": "%Y%m%d"}, + ], + ids=["s", "ms", "us", "ns", "yyyymmdd"], +) +def test_integer_time_column_uses_declared_unit(tmp_path, hint, sort_key, dtype): + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key, dtype + ) + recency = {"name": "r", "operation": "recency", "column": "event_time"} + result, lengths = run(engine, relation, [recency]) + + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == EXPECTED_IDS + assert list(sequence["timestamps"]) == [times[i - 1] for i in EXPECTED_IDS] + assert lengths["sequences"][0]["lengths"].to_pylist() == [len(EXPECTED_IDS)] + path = "sort_key_timestamps" if sort_key else "rowpack_timestamps" + assert lengths["stats"][path] == len(times) + assert list(result["aggregates"][0]["values"]) == [0.0] + + +@pytest.mark.parametrize("sort_key", [True, False], ids=["sort_key", "rowpack"]) +def test_sub_second_times_round_up_like_ingestion(tmp_path, sort_key): + lower = (CUTOFF - 3 * DAY) * 1000 + upper = CUTOFF * 1000 + event_times = [lower - 400, lower, lower + 400, upper - 400, upper + 400] + engine, relation = open_engine( + tmp_path, event_times, {"timestamp_unit": "millis"}, sort_key + ) + result, lengths = run(engine, relation) + + sequence = result["sequences"][0] + assert sequence["batch"].column("id").to_pylist() == [4, 3] + assert list(sequence["timestamps"]) == [CUTOFF, CUTOFF - 3 * DAY + 1] + assert lengths["sequences"][0]["lengths"].to_pylist() == [2] + path = "sort_key_timestamps" if sort_key else "rowpack_timestamps" + assert lengths["stats"][path] == len(event_times) + + +def test_grouped_recency_uses_declared_unit(tmp_path): + hint = {"timestamp_unit": "millis"} + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key=False + ) + count = {"name": "n", "operation": "count", "column": None} + latest = {"name": "m", "operation": "max", "column": "event_time"} + result, _ = run( + engine, + relation, + [ + { + "name": "by_time", + "groups": [{"keys": ["event_time"], "values": [count]}], + "operation": "recency", + "column": "event_time", + }, + { + "name": "of_max", + "groups": [{"keys": ["owner"], "values": [latest]}], + "operation": "recency", + "column": "m", + }, + { + "name": "grouped", + "groups": [ + { + "keys": ["owner"], + "values": [{**latest, "name": "r", "operation": "recency"}], + } + ], + "operation": "max", + "column": "r", + }, + { + "name": "two_stages", + "groups": [ + {"keys": ["id", "owner"], "values": [latest]}, + {"keys": ["owner"], "values": [{**latest, "column": "m"}]}, + ], + "operation": "recency", + "column": "m", + }, + { + "name": "overwritten_key", + "groups": [ + { + "keys": ["event_time"], + "values": [{**count, "name": "event_time"}], + } + ], + "operation": "recency", + "column": "event_time", + }, + {"name": "max", "operation": "max", "column": "event_time"}, + ], + ) + + values = {item["name"]: list(item["values"]) for item in result["aggregates"]} + assert values == { + "by_time": [0.0], + "of_max": [0.0], + "grouped": [0.0], + "two_stages": [0.0], + "overwritten_key": [float(CUTOFF - 1)], + "max": [encode(CUTOFF, hint)], + } + + +def test_unreadable_config_fails_temporal_queries(tmp_path): + engine, relation = open_engine( + tmp_path, [CUTOFF], {"timestamp_unit": "seconds"}, sort_key=True + ) + (tmp_path / "index" / "diskgraph_config.json").write_text("{") + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + + with pytest.raises(diskgraph.DiskgraphConfigError, match="diskgraph_config.json"): + run(engine, relation) + + +def test_unknown_config_fields_outside_time_columns_are_ignored(tmp_path): + hint = {"timestamp_unit": "millis"} + times = [CUTOFF - offset for offset in OFFSETS] + engine, relation = open_engine( + tmp_path, [encode(t, hint) for t in times], hint, sort_key=False + ) + path = tmp_path / "index" / "diskgraph_config.json" + config = json.loads(path.read_text()) + config["future_option"] = 1 + columns = config["tables"][1]["columns"] + columns[0]["ingested_dtype"] = "future_dtype" + columns[2].update(timestamp_format="%Y", semantic_type="future_stype") + columns[3]["semantic_type"] = "future_stype" + path.write_text(json.dumps(config)) + engine = diskgraph.SamplerEngine(str(tmp_path / "index")) + + result, _ = run(engine, relation) + assert result["sequences"][0]["batch"].column("id").to_pylist() == EXPECTED_IDS diff --git a/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs b/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs index 20f25663a..9f5b1ed19 100644 --- a/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs +++ b/packages/diskgraph/diskgraph-pybind/src/bp_col_stats.rs @@ -669,7 +669,7 @@ fn histogram_left_edge(spec: FloatingHistogramSpec, idx: usize) -> f64 { } } -fn row_access_hints(col: &ColumnSpec) -> RowAccessHints { +pub(crate) fn row_access_hints(col: &ColumnSpec) -> RowAccessHints { RowAccessHints { timestamp_format: col.timestamp_format.clone(), timestamp_unit: col.timestamp_unit.map(|unit| match unit { diff --git a/packages/diskgraph/diskgraph-pybind/src/lib.rs b/packages/diskgraph/diskgraph-pybind/src/lib.rs index e5c138b9a..6c4b46c70 100644 --- a/packages/diskgraph/diskgraph-pybind/src/lib.rs +++ b/packages/diskgraph/diskgraph-pybind/src/lib.rs @@ -493,6 +493,8 @@ impl OutputHandler for PyOutputHandler { #[pyclass] pub struct SamplerEngine { reader: Arc, + /// A config error is reported by temporal queries, not by sampling. + time_hints: Arc>, engine: Arc, pool: StatePool, sample_count: Arc, @@ -691,9 +693,14 @@ impl SamplerEngine { ); let engine = DiskGraphEngine::from_reader(Arc::clone(&reader), random_seed, num_threads) .map_err(map_engine_err)?; + let time_hints = reader + .time_columns() + .map_err(str::to_string) + .and_then(temporal_query::time_hints); info!("SamplerEngine initialized successfully"); Ok(Self { reader, + time_hints: Arc::new(time_hints), engine: Arc::new(engine), pool: StatePool::new(), sample_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), @@ -766,10 +773,12 @@ impl SamplerEngine { })?; let cutoffs: Vec = cutoff_values.extract()?; let reader = Arc::clone(&self.reader); + let time_hints = Arc::clone(&self.time_hints); let result = py .detach(move || { temporal_query::execute( reader.as_ref(), + time_hints.as_ref().as_ref().map_err(Clone::clone)?, &root_type, &roots, &cutoffs, @@ -811,10 +820,12 @@ impl SamplerEngine { })?; let cutoffs: Vec = cutoff_values.extract()?; let reader = Arc::clone(&self.reader); + let time_hints = Arc::clone(&self.time_hints); let result = py .detach(move || { temporal_query::execute_lengths( reader.as_ref(), + time_hints.as_ref().as_ref().map_err(Clone::clone)?, &root_type, &roots, &cutoffs, diff --git a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs index be9ba21af..1edb7c664 100644 --- a/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs +++ b/packages/diskgraph/diskgraph-pybind/src/temporal_query.rs @@ -25,7 +25,11 @@ use arrow_array::RecordBatch; use diskgraph_core::metadata::{EdgeRelationMetadata, EdgeSortKeyMetadata}; use diskgraph_core::sortkey::SortKeyDecoder; use diskgraph_engine::edge_unpacker::EdgeUnpacker; -use diskgraph_index::graph::{GraphIndexReader, TableReader}; +use diskgraph_index::graph::{GraphIndexReader, TableReader, TimeColumns}; +use diskgraph_ingest::graph_config::{ + ColumnSpec, IngestedType, TimestampUnit, DISKGRAPH_CONFIG_FILENAME, +}; +use diskgraph_rowpack::{RowAccess, RowAccessHints, RowAccessStats}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; use regex::Regex; @@ -541,6 +545,88 @@ fn scalar_at(array: &dyn Array, row: usize) -> Result { )) } +/// Ingest hints by column name, for one table. +type ColumnHints = HashMap; + +/// Ingest hints of integer columns declared with a time format or a unit other +/// than seconds, by table. +pub(crate) type TimeHints = HashMap; + +/// [`TimeHints`] of the time columns the index's persisted graph config +/// declares. Only the time fields of integer columns are parsed strictly, so +/// config fields this version does not know fail nothing else. +pub(crate) fn time_hints(tables: &TimeColumns) -> Result { + let mut hints = TimeHints::new(); + for (table, table_columns) in tables { + let mut columns = ColumnHints::new(); + for column in table_columns { + let Some(dtype) = column + .get("ingested_dtype") + .or_else(|| column.get("dtype")) + .filter(|dtype| { + matches!( + serde_json::from_value((*dtype).clone()), + Ok(IngestedType::Int8 + | IngestedType::Int16 + | IngestedType::Int32 + | IngestedType::Int64 + | IngestedType::UInt8 + | IngestedType::UInt16 + | IngestedType::UInt32 + | IngestedType::UInt64) + ) + }) + else { + continue; + }; + let mut fields = serde_json::Map::from_iter([("ingested_dtype".into(), dtype.clone())]); + for key in ["name", "timestamp_unit", "timestamp_format"] { + if let Some(value) = column.get(key) { + fields.insert(key.into(), value.clone()); + } + } + let column: ColumnSpec = serde_json::from_value(serde_json::Value::Object(fields)) + .map_err(|error| { + format!( + "parse {DISKGRAPH_CONFIG_FILENAME}: table {table:?} column {}: {error}", + column.get("name").unwrap_or(&serde_json::Value::Null) + ) + })?; + if column.timestamp_format.is_some() + || column + .timestamp_unit + .is_some_and(|unit| unit != TimestampUnit::Seconds) + { + let hint = crate::bp_col_stats::row_access_hints(&column); + columns.insert(column.name, hint); + } + } + hints.insert(table.clone(), columns); + } + Ok(hints) +} + +/// Epoch seconds of each value. A hinted integer column decodes exactly as +/// ingestion and its edge sort keys read it. +fn epoch_seconds( + array: &ArrayRef, + hint: Option<&RowAccessHints>, +) -> Result>, String> { + let Some(hint) = hint else { + return (0..array.len()) + .map(|row| scalar_at(array.as_ref(), row)?.epoch_seconds()) + .collect(); + }; + let mut access = RowAccess::new(vec![hint.clone()], RowAccessStats::new(None)); + access.set_cols(std::slice::from_ref(array)); + (0..array.len()) + .map(|row| { + access.set_row(row); + access.get_timestamp(0).map_err(|error| error.to_string()) + }) + .collect() +} + fn relation_columns( table: &TableReader, relation_name: &str, @@ -797,6 +883,7 @@ fn sort_frame_rows(rows: &mut [(usize, Option, ScalarValue)]) { fn materialize_frame( reader: &GraphIndexReader, + hints: &TimeHints, plan: &FramePlan, root_type: &str, roots: &[u64], @@ -925,7 +1012,14 @@ fn materialize_frame( input_offsets.push(input_offsets.last().copied().unwrap_or(0) + rows.len()); } let primary_key_position = positions[&primary_key]; - let time_position = plan.time_column.as_ref().map(|column| positions[column]); + let times = plan + .time_column + .as_ref() + .map(|column| { + let hint = column_hint(hints.get(&destination), Some(column)); + epoch_seconds(decoded.column(positions[column]), hint) + }) + .transpose()?; let mut selected_rows = Vec::::new(); let mut node_ids = Vec::new(); let mut timestamps = Vec::new(); @@ -940,16 +1034,13 @@ fn materialize_frame( .map(|value| cutoffs[anchor].saturating_add(value)); let mut anchor_rows = Vec::<(usize, Option, ScalarValue)>::new(); for row in input_offsets[anchor]..input_offsets[anchor + 1] { - let timestamp = match time_position { - Some(position) => Some( - scalar_at(decoded.column(position).as_ref(), row)? - .epoch_seconds()? - .ok_or_else(|| { - format!("frame {:?} contains a null timestamp", plan.name) - })?, - ), - None => None, - }; + let timestamp = times + .as_ref() + .map(|times| { + times[row] + .ok_or_else(|| format!("frame {:?} contains a null timestamp", plan.name)) + }) + .transpose()?; if timestamp.is_some_and(|value| { lower.is_some_and(|bound| value <= bound) || upper.is_some_and(|bound| value > bound) @@ -1124,6 +1215,13 @@ impl RowsView<'_> { } } +fn column_hint<'a>( + hints: Option<&'a ColumnHints>, + column: Option<&str>, +) -> Option<&'a RowAccessHints> { + hints?.get(column?) +} + fn validate_reduction(operation: &str, column: Option<&str>) -> Result<(), String> { if !REDUCTIONS.contains(&operation) { return Err(format!("unsupported temporal reduction {operation:?}")); @@ -1140,6 +1238,7 @@ fn reduce( column: Option<&str>, cutoff: i64, sum_type: Option, + hint: Option<&RowAccessHints>, ) -> Result { validate_reduction(operation, column)?; if sum_type.is_some() && operation != "sum" { @@ -1199,10 +1298,14 @@ fn reduce( } } if operation == "recency" { - let latest = values + let mut seconds = values .iter() .map(ScalarValue::epoch_seconds) - .collect::, _>>()? + .collect::, _>>()?; + if hint.is_some() { + seconds = epoch_seconds(&(Arc::new(Int64Array::from(seconds)) as ArrayRef), hint)?; + } + let latest = seconds .into_iter() .flatten() .max() @@ -1270,7 +1373,13 @@ fn reduce( }) } -fn group_rows(rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64) -> Result, String> { +/// `hints` describe the input columns. +fn group_rows( + rows: RowsView<'_>, + plan: &GroupPlan, + cutoff: i64, + hints: Option<&ColumnHints>, +) -> Result, String> { if plan.keys.is_empty() || plan.values.is_empty() { return Err("group_by requires keys and values".into()); } @@ -1337,6 +1446,7 @@ fn group_rows(rows: RowsView<'_>, plan: &GroupPlan, cutoff: i64) -> Result, plan: &GroupPlan, cutoff: i64) -> Result) -> ColumnHints { + let mut output = ColumnHints::new(); + for key in &plan.keys { + if let Some(hint) = column_hint(hints, Some(key)) { + output.insert(key.clone(), hint.clone()); + } + } + for value in &plan.values { + match column_hint(hints, value.column.as_deref()) + .filter(|_| matches!(value.operation.as_str(), "min" | "max" | "mode")) + { + Some(hint) => output.insert(value.name.clone(), hint.clone()), + None => output.remove(&value.name), + }; + } + output +} + fn aggregate_result( frame: &MaterializedFrame, + hints: Option<&ColumnHints>, plan: &AggregatePlan, cutoffs: &[i64], ) -> Result { validate_reduction(&plan.operation, plan.column.as_deref())?; + let mut stage_hints = Vec::with_capacity(plan.groups.len()); + for group in &plan.groups { + let output = group_output_hints(group, stage_hints.last().or(hints)); + stage_hints.push(output); + } let mut values = Vec::with_capacity(cutoffs.len()); for (anchor, cutoff) in cutoffs.iter().copied().enumerate() { let frame_view = RowsView::Frame { @@ -1359,9 +1495,9 @@ fn aggregate_result( stop: frame.offsets[anchor + 1], }; let value = if let Some((first, remaining)) = plan.groups.split_first() { - let mut grouped = group_rows(frame_view, first, cutoff)?; - for group in remaining { - grouped = group_rows(RowsView::Owned(&grouped), group, cutoff)?; + let mut grouped = group_rows(frame_view, first, cutoff, hints)?; + for (group, hints) in remaining.iter().zip(&stage_hints) { + grouped = group_rows(RowsView::Owned(&grouped), group, cutoff, Some(hints))?; } reduce( RowsView::Owned(&grouped), @@ -1369,6 +1505,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, + column_hint(stage_hints.last(), plan.column.as_deref()), )? } else { reduce( @@ -1377,6 +1514,7 @@ fn aggregate_result( plan.column.as_deref(), cutoff, plan.sum_type, + column_hint(hints, plan.column.as_deref()), )? }; values.push(value); @@ -1421,6 +1559,7 @@ fn aggregate_kind(frame: &MaterializedFrame, plan: &AggregatePlan) -> Option>(); let decoded = table_rows(table.as_ref(), &flat_nodes, &projection_indices)?; + let hint = column_hint(hints.get(&destination), Some(time_column)); + let times = epoch_seconds(decoded.column(0), hint)?; let mut row = 0; for (anchor, rows) in rows_by_anchor.iter().enumerate() { let lower = plan @@ -1522,11 +1663,9 @@ fn count_frame( .map(|value| cutoffs[anchor].saturating_add(value)); let mut selected = 0; for _ in rows { - let timestamp = scalar_at(decoded.column(0).as_ref(), row)? - .epoch_seconds()? - .ok_or_else(|| { - format!("frame {:?} contains a null timestamp", plan.name) - })?; + let timestamp = times[row].ok_or_else(|| { + format!("frame {:?} contains a null timestamp", plan.name) + })?; row += 1; stats.rowpack_timestamps += 1; if lower.is_none_or(|bound| timestamp > bound) @@ -1548,6 +1687,7 @@ fn count_frame( pub(crate) fn execute_lengths( reader: &GraphIndexReader, + hints: &TimeHints, root_type: &str, roots: &[u64], cutoffs: &[i64], @@ -1584,6 +1724,7 @@ pub(crate) fn execute_lengths( })?; let lengths = count_frame( reader, + hints, frame, root_type, roots, @@ -1615,6 +1756,7 @@ pub(crate) fn execute_lengths( pub(crate) fn execute( reader: &GraphIndexReader, + hints: &TimeHints, root_type: &str, roots: &[u64], cutoffs: &[i64], @@ -1679,6 +1821,7 @@ pub(crate) fn execute( }); let frame = materialize_frame( reader, + hints, frame_plan, root_type, roots, @@ -1720,7 +1863,7 @@ pub(crate) fn execute( aggregate.name, aggregate.frame ) })?; - aggregate_result(frame, aggregate, cutoffs) + aggregate_result(frame, hints.get(&frame.destination), aggregate, cutoffs) }) .collect::, String>>()?; timings.aggregation = elapsed_ns(aggregate_started); @@ -1961,7 +2104,7 @@ mod tests { fn reduction_empty_and_null_semantics_are_distinct() { let empty = Vec::::new(); assert!(matches!( - reduce(RowsView::Owned(&empty), "sum", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&empty), "sum", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(0.0) )); assert!(matches!( @@ -1970,7 +2113,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .unwrap(), ScalarValue::I64(0) @@ -1981,7 +2125,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .unwrap(), ScalarValue::Null @@ -1992,6 +2137,7 @@ mod tests { "count_distinct", Some("value"), 0, + None, None ) .unwrap(), @@ -2006,7 +2152,15 @@ mod tests { row(Some(ScalarValue::I64(1))), ]; assert!(matches!( - reduce(RowsView::Owned(&rows), "sum", Some("value"), 0, Some(SumType::Int64)).unwrap(), + reduce( + RowsView::Owned(&rows), + "sum", + Some("value"), + 0, + Some(SumType::Int64), + None + ) + .unwrap(), ScalarValue::I64(value) if value == (1 << 53) + 1 )); let output = scalar_values_to_array( @@ -2043,7 +2197,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Int64) + Some(SumType::Int64), + None ) .is_err()); let unsigned = [row(Some(ScalarValue::U64(u64::MAX)))]; @@ -2053,7 +2208,8 @@ mod tests { "sum", Some("value"), 0, - Some(SumType::Uint64) + Some(SumType::Uint64), + None ) .unwrap(), ScalarValue::U64(u64::MAX) @@ -2063,7 +2219,15 @@ mod tests { row(Some(ScalarValue::F64(2.5))), ]; assert!(matches!( - reduce(RowsView::Owned(&floats), "sum", Some("value"), 0, None).unwrap(), + reduce( + RowsView::Owned(&floats), + "sum", + Some("value"), + 0, + None, + None + ) + .unwrap(), ScalarValue::F64(3.75) )); } @@ -2076,11 +2240,11 @@ mod tests { row(Some(ScalarValue::F64(30.0))), ]; assert!(matches!( - reduce(RowsView::Owned(&rows), "var_population", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&rows), "var_population", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(value) if (value - 66.66666666666667).abs() < 1e-10 )); assert!(matches!( - reduce(RowsView::Owned(&rows), "std_sample", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&rows), "std_sample", Some("value"), 0, None, None).unwrap(), ScalarValue::F64(value) if (value - 10.0).abs() < 1e-10 )); let tied = vec![ @@ -2088,7 +2252,7 @@ mod tests { row(Some(ScalarValue::String("a".into()))), ]; assert!(matches!( - reduce(RowsView::Owned(&tied), "mode", Some("value"), 0, None).unwrap(), + reduce(RowsView::Owned(&tied), "mode", Some("value"), 0, None, None).unwrap(), ScalarValue::String(value) if value == "a" )); }