From 2624ffcac69d980c220b116fdbc393120fbefdbe Mon Sep 17 00:00:00 2001 From: Min Shen <4633527+Victsm@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:19:56 +0000 Subject: [PATCH 1/2] Use smaller native work partitions for history minibatches --- .../src/batch_aggregate.rs | 9 ++- .../tests/ttgen_integration.rs | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/packages/diskgraph/diskgraph-train-table/src/batch_aggregate.rs b/packages/diskgraph/diskgraph-train-table/src/batch_aggregate.rs index 7e50a33c3..291a0dbea 100644 --- a/packages/diskgraph/diskgraph-train-table/src/batch_aggregate.rs +++ b/packages/diskgraph/diskgraph-train-table/src/batch_aggregate.rs @@ -2344,14 +2344,19 @@ fn execute_prepared_parallel( plan: &QueryPlan, pool: &rayon::ThreadPool, ) -> Result { - if roots.len() != cutoffs.len() || roots.len() < 64 || pool.current_num_threads() == 1 { + let anchors_per_chunk = if plan.aggregates.is_empty() && !plan.sequences.is_empty() { + 8 + } else { + 64 + }; + if roots.len() != cutoffs.len() || roots.len() < anchors_per_chunk || pool.current_num_threads() == 1 { if roots.len() != cutoffs.len() { return Err("root IDs and cutoffs must have equal length".into()); } return execute_query(reader, root_type, roots, cutoffs, plan); } let started = Instant::now(); - let chunks = roots.len().div_ceil(64).min(pool.current_num_threads() * 2); + let chunks = roots.len().div_ceil(anchors_per_chunk).min(pool.current_num_threads() * 2); let size = roots.len().div_ceil(chunks); let results = pool.install(|| { roots diff --git a/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs b/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs index 1139f0643..823325542 100644 --- a/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs +++ b/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs @@ -10661,3 +10661,63 @@ fn integ_prepared_history_reuses_frames_and_binds_clock_before_iteration() { assert!(untimed.require_timed_sequence("history").unwrap_err().contains("timed frames")); assert!(untimed.execute(&reader, &[0], &[0], &pool).is_ok()); } + +#[test] +fn integ_parallel_history_preserves_ragged_batches() { + use diskgraph_index::pkmap::Key; + use diskgraph_train_table::batch_aggregate::PreparedTemporalPlan; + + let dir = tempfile::tempdir().unwrap(); + let index = build_graph1(dir.path()); + let reader = diskgraph_index::graph::GraphIndexReader::open( + &index.join("index_metadata.json"), &index, + ).unwrap(); + let users = reader.table("users").unwrap(); + let ids = (0..5).map(|id| users.node_id_from_pk(&Key::Int(id)).unwrap().unwrap()) + .collect::>(); + let relation = reader.metadata().edge_relations.iter() + .find(|r| r.src_type == "users" && r.dst_type == "orders").unwrap().name.clone(); + let pools = [1, 2, 8].map(|threads| { + rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap() + }); + for limit in [None, Some(1)] { + let json = serde_json::json!({ + "frames": [{"name":"events", "root_type":"users", + "path":[{"relation":relation,"destination_type":"orders"}], + "time_column":"ts", "start_seconds":null,"stop_seconds":0, + "distinct":true,"columns":["price","status"]}], + "sequences":[ + {"name":"history","frame":"events","columns":["price","status"], + "order":"ascending","limit":limit}, + {"name":"recent","frame":"events","columns":["status"], + "order":"descending","limit":limit}] + }); + let prepared = PreparedTemporalPlan::prepare_history( + &reader, "users", &json.to_string(), + ).unwrap(); + for size in [0, 1, 7, 8, 31, 32, 63, 64, 65, 128, 256] { + for repeated in [false, true] { + let roots = (0..size).map(|i| ids[if repeated { 0 } else { i % ids.len() }]) + .collect::>(); + for empty in [false, true] { + // An empty prefix checks concatenation when the first job + // has no rows but later jobs do; roots may repeat freely. + let cutoffs = (0..size).map(|i| if empty || i < 8 { 0 } else { 5000 }) + .collect::>(); + let expected = prepared.execute(&reader, &roots, &cutoffs, &pools[0]).unwrap(); + for pool in &pools[1..] { + let actual = prepared.execute(&reader, &roots, &cutoffs, pool).unwrap(); + assert_eq!(actual.sequences.len(), expected.sequences.len()); + for (actual, expected) in actual.sequences.iter().zip(&expected.sequences) { + assert_eq!(actual.name, expected.name); + assert_eq!(actual.offsets, expected.offsets); + assert_eq!(actual.node_ids, expected.node_ids); + assert_eq!(actual.timestamps, expected.timestamps); + assert_eq!(actual.batch, expected.batch); + } + } + } + } + } + } +} From 882dbdd4ba00266cc4ccebb137a33c63ba876a6f Mon Sep 17 00:00:00 2001 From: Min Shen <4633527+Victsm@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:43:09 +0000 Subject: [PATCH 2/2] Assert history partition results contain expected capped rows --- .../tests/ttgen_integration.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs b/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs index 823325542..0737a86c0 100644 --- a/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs +++ b/packages/diskgraph/diskgraph-train-table/tests/ttgen_integration.rs @@ -10705,15 +10705,26 @@ fn integ_parallel_history_preserves_ragged_batches() { let cutoffs = (0..size).map(|i| if empty || i < 8 { 0 } else { 5000 }) .collect::>(); let expected = prepared.execute(&reader, &roots, &cutoffs, &pools[0]).unwrap(); + let context = format!("limit={limit:?} size={size} repeated={repeated} empty={empty}"); + for sequence in &expected.sequences { + for i in 0..size { + let rows = if empty || i < 8 { 0 } else if repeated { 4 } else { [4, 2, 1, 3, 0][i % 5] }; + let rows = limit.map_or(rows, |maximum| rows.min(maximum)); + assert_eq!(sequence.offsets[i + 1] - sequence.offsets[i], rows, "{context} anchor={i}"); + } + if !empty && size > 8 { + assert!(!sequence.node_ids.is_empty(), "{context}"); + } + } for pool in &pools[1..] { let actual = prepared.execute(&reader, &roots, &cutoffs, pool).unwrap(); - assert_eq!(actual.sequences.len(), expected.sequences.len()); + assert_eq!(actual.sequences.len(), expected.sequences.len(), "{context}"); for (actual, expected) in actual.sequences.iter().zip(&expected.sequences) { - assert_eq!(actual.name, expected.name); - assert_eq!(actual.offsets, expected.offsets); - assert_eq!(actual.node_ids, expected.node_ids); - assert_eq!(actual.timestamps, expected.timestamps); - assert_eq!(actual.batch, expected.batch); + assert_eq!(actual.name, expected.name, "{context}"); + assert_eq!(actual.offsets, expected.offsets, "{context}"); + assert_eq!(actual.node_ids, expected.node_ids, "{context}"); + assert_eq!(actual.timestamps, expected.timestamps, "{context}"); + assert_eq!(actual.batch, expected.batch, "{context}"); } } }