Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2346,14 +2346,19 @@ fn execute_prepared_parallel(
plan: &QueryPlan,
pool: &rayon::ThreadPool,
) -> Result<QueryResult, String> {
if roots.len() != cutoffs.len() || roots.len() < 64 || pool.current_num_threads() == 1 {
let anchors_per_chunk = if plan.aggregates.is_empty() && !plan.sequences.is_empty() {
8
} else {
64
};
if roots.len() != cutoffs.len() || roots.len() < anchors_per_chunk || pool.current_num_threads() == 1 {
if roots.len() != cutoffs.len() {
return Err("root IDs and cutoffs must have equal length".into());
}
return execute_query(reader, root_type, roots, cutoffs, plan);
}
let started = Instant::now();
let chunks = roots.len().div_ceil(64).min(pool.current_num_threads() * 2);
let chunks = roots.len().div_ceil(anchors_per_chunk).min(pool.current_num_threads() * 2);
let size = roots.len().div_ceil(chunks);
let results = pool.install(|| {
roots
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10661,3 +10661,74 @@ fn integ_prepared_history_reuses_frames_and_binds_clock_before_iteration() {
assert!(untimed.require_timed_sequence("history").unwrap_err().contains("timed frames"));
assert!(untimed.execute(&reader, &[0], &[0], &pool).is_ok());
}

#[test]
fn integ_parallel_history_preserves_ragged_batches() {
use diskgraph_index::pkmap::Key;
use diskgraph_train_table::batch_aggregate::PreparedTemporalPlan;

let dir = tempfile::tempdir().unwrap();
let index = build_graph1(dir.path());
let reader = diskgraph_index::graph::GraphIndexReader::open(
&index.join("index_metadata.json"), &index,
).unwrap();
let users = reader.table("users").unwrap();
let ids = (0..5).map(|id| users.node_id_from_pk(&Key::Int(id)).unwrap().unwrap())
.collect::<Vec<_>>();
let relation = reader.metadata().edge_relations.iter()
.find(|r| r.src_type == "users" && r.dst_type == "orders").unwrap().name.clone();
let pools = [1, 2, 8].map(|threads| {
rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap()
});
for limit in [None, Some(1)] {
let json = serde_json::json!({
"frames": [{"name":"events", "root_type":"users",
"path":[{"relation":relation,"destination_type":"orders"}],
"time_column":"ts", "start_seconds":null,"stop_seconds":0,
"distinct":true,"columns":["price","status"]}],
"sequences":[
{"name":"history","frame":"events","columns":["price","status"],
"order":"ascending","limit":limit},
{"name":"recent","frame":"events","columns":["status"],
"order":"descending","limit":limit}]
});
let prepared = PreparedTemporalPlan::prepare_history(
&reader, "users", &json.to_string(),
).unwrap();
for size in [0, 1, 7, 8, 31, 32, 63, 64, 65, 128, 256] {
for repeated in [false, true] {
let roots = (0..size).map(|i| ids[if repeated { 0 } else { i % ids.len() }])
.collect::<Vec<_>>();
for empty in [false, true] {
// An empty prefix checks concatenation when the first job
// has no rows but later jobs do; roots may repeat freely.
let cutoffs = (0..size).map(|i| if empty || i < 8 { 0 } else { 5000 })
.collect::<Vec<_>>();
let expected = prepared.execute(&reader, &roots, &cutoffs, &pools[0]).unwrap();
let context = format!("limit={limit:?} size={size} repeated={repeated} empty={empty}");
for sequence in &expected.sequences {
for i in 0..size {
let rows = if empty || i < 8 { 0 } else if repeated { 4 } else { [4, 2, 1, 3, 0][i % 5] };
let rows = limit.map_or(rows, |maximum| rows.min(maximum));
assert_eq!(sequence.offsets[i + 1] - sequence.offsets[i], rows, "{context} anchor={i}");
}
if !empty && size > 8 {
assert!(!sequence.node_ids.is_empty(), "{context}");
}
}
for pool in &pools[1..] {
let actual = prepared.execute(&reader, &roots, &cutoffs, pool).unwrap();
assert_eq!(actual.sequences.len(), expected.sequences.len(), "{context}");
for (actual, expected) in actual.sequences.iter().zip(&expected.sequences) {
assert_eq!(actual.name, expected.name, "{context}");
assert_eq!(actual.offsets, expected.offsets, "{context}");
assert_eq!(actual.node_ids, expected.node_ids, "{context}");
assert_eq!(actual.timestamps, expected.timestamps, "{context}");
assert_eq!(actual.batch, expected.batch, "{context}");
}
}
}
}
}
}
}
Loading