From 47e969ac188e773ee62b2769079b5219d960ae31 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:24:26 +0100 Subject: [PATCH 01/31] refactor: move benchmark dataset generation into datafusion-datasets crate (#866) Breaks the dependency cycle between `datafusion-distributed` and `datafusion-distributed-benchmarks`. Before this change, the core crate dev-depended on the benchmarks crate (for TPC-H/TPC-DS/ClickBench data in tests), and the benchmarks crate depended on the core crate. This implied that for every change in `datafusion-distributed`, the `datafusion-distributed-benchmarks` needed to be recompiled (as it's dependant) and then `datafusion-distributed` itself needs to be re-recompiled again, causing compilation times to be very slow. Here are some numbers of adding a small trivial change to one test before vs after: ## Before (32.58s) ``` time cargo test --lib test_select_all Compiling datafusion-distributed v4.0.0 (//datafusion-distributed/datafusion/distributed) Compiling datafusion-distributed-benchmarks v0.1.0 (//datafusion-distributed/datafusion/distributed-benchmarks) Finished `test` profile [unoptimized + debuginfo] target(s) in 32.58s ``` ## After (17.01s) ``` time cargo test --lib test_select_all Compiling datafusion-distributed v4.0.0 (//datafusion-distributed/datafusion/distributed) Finished `test` profile [unoptimized + debuginfo] target(s) in 17.01s ``` So for testing targets, this cuts compilation time in half --- AGENTS.md | 3 ++ Cargo.lock | 39 +++++++++++++------ Cargo.toml | 1 + datafusion/datasets/Cargo.toml | 25 ++++++++++++ .../datasets => datasets/src}/clickbench.rs | 4 +- .../src/datasets => datasets/src}/common.rs | 4 +- .../datasets/mod.rs => datasets/src/lib.rs} | 3 ++ .../src/datasets => datasets/src}/output.rs | 2 +- .../src}/parquet_layout.rs | 2 +- .../src/prepare_clickbench.rs | 4 +- .../src/prepare_tpcds.rs | 6 +-- .../src/prepare_tpch.rs | 6 +-- .../src/datasets => datasets/src}/progress.rs | 0 .../src/datasets => datasets/src}/tpcds.rs | 6 +-- .../src/datasets => datasets/src}/tpch.rs | 6 +-- datafusion/distributed-benchmarks/Cargo.toml | 11 +----- datafusion/distributed-benchmarks/README.md | 2 +- .../benches/metrics_collection.rs | 8 ++-- .../benches/planning.rs | 8 ++-- .../distributed-benchmarks/src/backend.rs | 2 +- datafusion/distributed-benchmarks/src/lib.rs | 1 - datafusion/distributed-benchmarks/src/main.rs | 4 +- datafusion/distributed-benchmarks/src/run.rs | 2 +- datafusion/distributed-console/Cargo.toml | 2 +- .../examples/tpcds_runner.rs | 6 +-- .../distributed-iceberg-benchmarks/Cargo.toml | 1 + .../src/dataset.rs | 2 +- .../src/prepare.rs | 2 +- datafusion/distributed/Cargo.toml | 2 +- datafusion/distributed/tests/chaos.rs | 6 +-- .../tests/clickbench_correctness_test.rs | 4 +- .../tests/clickbench_plans_test.rs | 4 +- datafusion/distributed/tests/shuffle.rs | 6 +-- .../tests/stateful_data_cleanup.rs | 6 +-- .../tests/tpcds_correctness_test.rs | 10 ++--- .../distributed/tests/tpcds_plans_test.rs | 10 ++--- .../tests/tpch_correctness_test.rs | 6 +-- .../distributed/tests/tpch_plans_test.rs | 6 +-- .../tests/tpch_sorted_correctness_test.rs | 6 +-- .../tests/tpch_sorted_plans_test.rs | 6 +-- 40 files changed, 133 insertions(+), 101 deletions(-) create mode 100644 datafusion/datasets/Cargo.toml rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/clickbench.rs (98%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/common.rs (93%) rename datafusion/{distributed-benchmarks/src/datasets/mod.rs => datasets/src/lib.rs} (68%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/output.rs (99%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/parquet_layout.rs (99%) rename datafusion/{distributed-benchmarks => datasets}/src/prepare_clickbench.rs (89%) rename datafusion/{distributed-benchmarks => datasets}/src/prepare_tpcds.rs (81%) rename datafusion/{distributed-benchmarks => datasets}/src/prepare_tpch.rs (88%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/progress.rs (100%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/tpcds.rs (98%) rename datafusion/{distributed-benchmarks/src/datasets => datasets/src}/tpch.rs (96%) diff --git a/AGENTS.md b/AGENTS.md index 6a47d8238..96365910b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,9 @@ about a user's networking or deployment stack. - `datafusion/distributed-iceberg-test-tools/`: shared Iceberg test fixtures. - `datafusion/distributed-console/`: terminal console for observing a cluster. - `datafusion/distributed-cli/`: distributed variant of the DataFusion CLI. +- `datafusion/datasets/`: TPC-H, TPC-DS and ClickBench + dataset generation shared by tests and benchmarks; it must not depend on + `datafusion-distributed`. - `datafusion/distributed-benchmarks/`, `datafusion/distributed-iceberg-benchmarks/`, `datafusion/distributed-remote-worker/`: local benchmark crates, remote benchmark worker, and microbenchmarks; do not change benchmark behavior as an diff --git a/Cargo.lock b/Cargo.lock index 762d2cf2f..d8fcf1fcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2211,6 +2211,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-datasets" +version = "0.1.0" +dependencies = [ + "arrow", + "aws-config", + "aws-credential-types", + "datafusion", + "futures", + "indicatif", + "object_store", + "parquet", + "reqwest 0.12.28", + "structopt", + "tempfile", + "tokio", + "tpcdsgen", + "tpcdsgen-arrow", + "tpchgen", + "tpchgen-arrow", + "url", +] + [[package]] name = "datafusion-datasource" version = "55.0.0" @@ -2387,7 +2410,7 @@ dependencies = [ "crossbeam-queue", "dashmap", "datafusion", - "datafusion-distributed-benchmarks", + "datafusion-datasets", "datafusion-proto", "delegate", "futures", @@ -2423,29 +2446,20 @@ version = "0.1.0" dependencies = [ "arrow", "async-trait", - "aws-config", - "aws-credential-types", "built", "criterion", "datafusion", + "datafusion-datasets", "datafusion-distributed", "env_logger", "futures", - "indicatif", - "object_store", "parquet", - "reqwest 0.12.28", "serde", "serde_json", "sketches-ddsketch", "structopt", - "tempfile", "tokio", "tonic", - "tpcdsgen", - "tpcdsgen-arrow", - "tpchgen", - "tpchgen-arrow", "url", ] @@ -2471,8 +2485,8 @@ dependencies = [ "color-eyre", "crossterm", "datafusion", + "datafusion-datasets", "datafusion-distributed", - "datafusion-distributed-benchmarks", "futures", "ratatui", "structopt", @@ -2512,6 +2526,7 @@ version = "0.1.0" dependencies = [ "async-trait", "datafusion", + "datafusion-datasets", "datafusion-distributed-benchmarks", "datafusion-distributed-iceberg", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index de8d9fceb..2ef245005 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "datafusion/datasets", "datafusion/distributed", "datafusion/distributed-benchmarks", "datafusion/distributed-cli", diff --git a/datafusion/datasets/Cargo.toml b/datafusion/datasets/Cargo.toml new file mode 100644 index 000000000..68a074255 --- /dev/null +++ b/datafusion/datasets/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "datafusion-datasets" +version = "0.1.0" +edition = "2024" +description = "TPC-H, TPC-DS and ClickBench dataset generation for DataFusion Distributed benchmarks and tests" +publish = false + +[dependencies] +datafusion = { workspace = true, features = ["parquet"] } +tokio = { version = "1.48", features = ["full"] } +parquet = { version = "59" } +structopt = { version = "0.3.26" } +futures = "0.3.31" +indicatif = "0.18" +arrow = { version = "59" } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } +tpcdsgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } +tpcdsgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } +reqwest = "0.12" +object_store = { version = "0.13", features = ["aws"] } +aws-config = "1" +aws-credential-types = "1" +url = "2" +tempfile = "3" diff --git a/datafusion/distributed-benchmarks/src/datasets/clickbench.rs b/datafusion/datasets/src/clickbench.rs similarity index 98% rename from datafusion/distributed-benchmarks/src/datasets/clickbench.rs rename to datafusion/datasets/src/clickbench.rs index b52f9f2d0..2f120ea10 100644 --- a/datafusion/distributed-benchmarks/src/datasets/clickbench.rs +++ b/datafusion/datasets/src/clickbench.rs @@ -1,5 +1,5 @@ -use super::progress::dataset_progress; -use super::{common, output::DatasetOutput}; +use crate::progress::dataset_progress; +use crate::{common, output::DatasetOutput}; use arrow::array::Array; use arrow::compute::cast; use arrow::datatypes::{DataType, Field, Schema}; diff --git a/datafusion/distributed-benchmarks/src/datasets/common.rs b/datafusion/datasets/src/common.rs similarity index 93% rename from datafusion/distributed-benchmarks/src/datasets/common.rs rename to datafusion/datasets/src/common.rs index aa53ed1f7..62416bf39 100644 --- a/datafusion/distributed-benchmarks/src/datasets/common.rs +++ b/datafusion/datasets/src/common.rs @@ -3,12 +3,12 @@ use datafusion::prelude::{ParquetReadOptions, SessionContext}; use std::fs; use std::path::Path; -/// Returns the workspace root directory (two levels above the benchmarks crate). +/// Returns the workspace root directory (two levels above this crate). pub(crate) fn workspace_root() -> &'static Path { Path::new(env!("CARGO_MANIFEST_DIR")) .ancestors() .nth(2) - .expect("benchmarks crate should be at datafusion/distributed-benchmarks in the workspace") + .expect("datasets crate should be at datafusion/datasets in the workspace") } pub fn get_queries(path: &str) -> Vec { diff --git a/datafusion/distributed-benchmarks/src/datasets/mod.rs b/datafusion/datasets/src/lib.rs similarity index 68% rename from datafusion/distributed-benchmarks/src/datasets/mod.rs rename to datafusion/datasets/src/lib.rs index 3cc5a8533..c3b37f91b 100644 --- a/datafusion/distributed-benchmarks/src/datasets/mod.rs +++ b/datafusion/datasets/src/lib.rs @@ -2,6 +2,9 @@ pub mod clickbench; pub(crate) mod common; pub mod output; pub mod parquet_layout; +pub mod prepare_clickbench; +pub mod prepare_tpcds; +pub mod prepare_tpch; mod progress; pub mod tpcds; pub mod tpch; diff --git a/datafusion/distributed-benchmarks/src/datasets/output.rs b/datafusion/datasets/src/output.rs similarity index 99% rename from datafusion/distributed-benchmarks/src/datasets/output.rs rename to datafusion/datasets/src/output.rs index 978cd6a1b..9f8f3ecaa 100644 --- a/datafusion/distributed-benchmarks/src/datasets/output.rs +++ b/datafusion/datasets/src/output.rs @@ -14,7 +14,7 @@ use futures::TryStreamExt; use tokio::io::AsyncWriteExt; use url::Url; -use crate::datasets::common::workspace_root; +use crate::common::workspace_root; const SUCCESS: &str = "_SUCCESS"; diff --git a/datafusion/distributed-benchmarks/src/datasets/parquet_layout.rs b/datafusion/datasets/src/parquet_layout.rs similarity index 99% rename from datafusion/distributed-benchmarks/src/datasets/parquet_layout.rs rename to datafusion/datasets/src/parquet_layout.rs index 608be8313..01343aa46 100644 --- a/datafusion/distributed-benchmarks/src/datasets/parquet_layout.rs +++ b/datafusion/datasets/src/parquet_layout.rs @@ -1,4 +1,4 @@ -use super::output::DatasetOutput; +use crate::output::DatasetOutput; use arrow::array::RecordBatch; use arrow::compute::cast; use arrow::datatypes::{DataType, Field, Schema}; diff --git a/datafusion/distributed-benchmarks/src/prepare_clickbench.rs b/datafusion/datasets/src/prepare_clickbench.rs similarity index 89% rename from datafusion/distributed-benchmarks/src/prepare_clickbench.rs rename to datafusion/datasets/src/prepare_clickbench.rs index 1626cdcf3..6e966cdda 100644 --- a/datafusion/distributed-benchmarks/src/prepare_clickbench.rs +++ b/datafusion/datasets/src/prepare_clickbench.rs @@ -1,7 +1,7 @@ use datafusion::common::{Result, exec_datafusion_err}; -use datafusion_distributed_benchmarks::datasets::clickbench::{ClickBenchSpec, generate_data}; -use datafusion_distributed_benchmarks::datasets::output::TESTDATA_ROOT; +use crate::clickbench::{ClickBenchSpec, generate_data}; +use crate::output::TESTDATA_ROOT; use structopt::StructOpt; /// Prepare ClickBench parquet files for benchmarks diff --git a/datafusion/distributed-benchmarks/src/prepare_tpcds.rs b/datafusion/datasets/src/prepare_tpcds.rs similarity index 81% rename from datafusion/distributed-benchmarks/src/prepare_tpcds.rs rename to datafusion/datasets/src/prepare_tpcds.rs index 094a8b738..4426f2442 100644 --- a/datafusion/distributed-benchmarks/src/prepare_tpcds.rs +++ b/datafusion/datasets/src/prepare_tpcds.rs @@ -1,7 +1,7 @@ +use crate::output::TESTDATA_ROOT; +use crate::parquet_layout::ParquetLayout; +use crate::tpcds::{TpcdsSpec, generate_data}; use datafusion::common::{Result, exec_datafusion_err}; -use datafusion_distributed_benchmarks::datasets::output::TESTDATA_ROOT; -use datafusion_distributed_benchmarks::datasets::parquet_layout::ParquetLayout; -use datafusion_distributed_benchmarks::datasets::tpcds::{TpcdsSpec, generate_data}; use structopt::StructOpt; /// Prepare TPC-DS parquet files for benchmarks diff --git a/datafusion/distributed-benchmarks/src/prepare_tpch.rs b/datafusion/datasets/src/prepare_tpch.rs similarity index 88% rename from datafusion/distributed-benchmarks/src/prepare_tpch.rs rename to datafusion/datasets/src/prepare_tpch.rs index c9f2f1cd2..23a1afd05 100644 --- a/datafusion/distributed-benchmarks/src/prepare_tpch.rs +++ b/datafusion/datasets/src/prepare_tpch.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. +use crate::output::TESTDATA_ROOT; +use crate::parquet_layout::ParquetLayout; +use crate::tpch::{TpchSpec, generate_data}; use datafusion::common::{Result, exec_datafusion_err}; -use datafusion_distributed_benchmarks::datasets::output::TESTDATA_ROOT; -use datafusion_distributed_benchmarks::datasets::parquet_layout::ParquetLayout; -use datafusion_distributed_benchmarks::datasets::tpch::{TpchSpec, generate_data}; use structopt::StructOpt; /// Generate TPC-H parquet files for benchmarks diff --git a/datafusion/distributed-benchmarks/src/datasets/progress.rs b/datafusion/datasets/src/progress.rs similarity index 100% rename from datafusion/distributed-benchmarks/src/datasets/progress.rs rename to datafusion/datasets/src/progress.rs diff --git a/datafusion/distributed-benchmarks/src/datasets/tpcds.rs b/datafusion/datasets/src/tpcds.rs similarity index 98% rename from datafusion/distributed-benchmarks/src/datasets/tpcds.rs rename to datafusion/datasets/src/tpcds.rs index 60ffffc32..2977d80ab 100644 --- a/datafusion/distributed-benchmarks/src/datasets/tpcds.rs +++ b/datafusion/datasets/src/tpcds.rs @@ -1,6 +1,6 @@ -use super::parquet_layout::{ParquetLayout, TableColumns, TableWriter}; -use super::progress::dataset_progress; -use super::{common, output::DatasetOutput}; +use crate::parquet_layout::{ParquetLayout, TableColumns, TableWriter}; +use crate::progress::dataset_progress; +use crate::{common, output::DatasetOutput}; use arrow::datatypes::{Field, Schema}; use arrow::record_batch::RecordBatch; use datafusion::common::exec_datafusion_err; diff --git a/datafusion/distributed-benchmarks/src/datasets/tpch.rs b/datafusion/datasets/src/tpch.rs similarity index 96% rename from datafusion/distributed-benchmarks/src/datasets/tpch.rs rename to datafusion/datasets/src/tpch.rs index bb9f52076..41af6dfd1 100644 --- a/datafusion/distributed-benchmarks/src/datasets/tpch.rs +++ b/datafusion/datasets/src/tpch.rs @@ -1,6 +1,6 @@ -use super::parquet_layout::{ParquetLayout, TableColumns, TableWriter}; -use super::progress::dataset_progress; -use super::{common, output::DatasetOutput}; +use crate::parquet_layout::{ParquetLayout, TableColumns, TableWriter}; +use crate::progress::dataset_progress; +use crate::{common, output::DatasetOutput}; use datafusion::common::{DataFusionError, exec_datafusion_err}; use indicatif::HumanCount; use tpchgen::generators::{ diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index 08e548e02..866b650c6 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -7,6 +7,7 @@ default-run = "dfbench" [dependencies] async-trait = "0.1.89" datafusion = { workspace = true } +datafusion-datasets = { path = "../datasets" } datafusion-distributed = { path = "../distributed", features = [ "integration", "system-metrics", @@ -18,20 +19,10 @@ serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.141" env_logger = "0.11.8" futures = "0.3.31" -indicatif = "0.18" arrow = { version = "59", features = ["test_utils"] } tonic = { version = "0.14.1", features = ["transport"] } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } -tpcdsgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } -tpcdsgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "f42708c32e7fd816b8ad8d2a880c1802a71b669f" } -reqwest = "0.12" sketches-ddsketch = "0.3" -object_store = { version = "0.13", features = ["aws"] } -aws-config = "1" -aws-credential-types = "1" url = "2" -tempfile = "3" [dev-dependencies] criterion = "0.5" diff --git a/datafusion/distributed-benchmarks/README.md b/datafusion/distributed-benchmarks/README.md index d5c01fb6f..21aa8cd81 100644 --- a/datafusion/distributed-benchmarks/README.md +++ b/datafusion/distributed-benchmarks/README.md @@ -42,7 +42,7 @@ options: With `--sorted`, the sort keys declared for each table are listed in `table_columns` in -`src/datasets/tpch.rs` and `tpcds.rs`. +`../datasets/src/tpch.rs` and `tpcds.rs`. `_SUCCESS` is written once every table is complete, and rerunning a `prepare-*` command for a complete dataset does nothing. Any other non-empty destination, diff --git a/datafusion/distributed-benchmarks/benches/metrics_collection.rs b/datafusion/distributed-benchmarks/benches/metrics_collection.rs index cbbe88466..b24177a4f 100644 --- a/datafusion/distributed-benchmarks/benches/metrics_collection.rs +++ b/datafusion/distributed-benchmarks/benches/metrics_collection.rs @@ -1,15 +1,15 @@ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use datafusion::error::DataFusionError; use datafusion::physical_plan::collect; +use datafusion_datasets::output::TESTDATA_ROOT; +use datafusion_datasets::parquet_layout::ParquetLayout; +use datafusion_datasets::tpch::TpchSpec; +use datafusion_datasets::{register_tables, tpch}; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedMetricsFormat, display_plan_ascii, rewrite_distributed_plan_with_metrics, }; -use datafusion_distributed_benchmarks::datasets::output::TESTDATA_ROOT; -use datafusion_distributed_benchmarks::datasets::parquet_layout::ParquetLayout; -use datafusion_distributed_benchmarks::datasets::tpch::TpchSpec; -use datafusion_distributed_benchmarks::datasets::{register_tables, tpch}; use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::Builder as RuntimeBuilder; diff --git a/datafusion/distributed-benchmarks/benches/planning.rs b/datafusion/distributed-benchmarks/benches/planning.rs index f8a30e67e..d2b530e2e 100644 --- a/datafusion/distributed-benchmarks/benches/planning.rs +++ b/datafusion/distributed-benchmarks/benches/planning.rs @@ -1,11 +1,11 @@ use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use datafusion::prelude::SessionContext; +use datafusion_datasets::output::TESTDATA_ROOT; +use datafusion_datasets::parquet_layout::ParquetLayout; +use datafusion_datasets::tpch::TpchSpec; +use datafusion_datasets::{register_tables, tpch}; use datafusion_distributed::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; use datafusion_distributed::{DistributedExt, create_distributed_plan}; -use datafusion_distributed_benchmarks::datasets::output::TESTDATA_ROOT; -use datafusion_distributed_benchmarks::datasets::parquet_layout::ParquetLayout; -use datafusion_distributed_benchmarks::datasets::tpch::TpchSpec; -use datafusion_distributed_benchmarks::datasets::{register_tables, tpch}; use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::Builder as RuntimeBuilder; diff --git a/datafusion/distributed-benchmarks/src/backend.rs b/datafusion/distributed-benchmarks/src/backend.rs index 9bdbadbc7..08115fb45 100644 --- a/datafusion/distributed-benchmarks/src/backend.rs +++ b/datafusion/distributed-benchmarks/src/backend.rs @@ -5,7 +5,7 @@ use datafusion::error::Result; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::SessionContext; -use crate::datasets::register_tables; +use datafusion_datasets::register_tables; #[async_trait] pub trait BenchmarkBackend: Send + Sync + 'static { diff --git a/datafusion/distributed-benchmarks/src/lib.rs b/datafusion/distributed-benchmarks/src/lib.rs index 345ba9520..41142c668 100644 --- a/datafusion/distributed-benchmarks/src/lib.rs +++ b/datafusion/distributed-benchmarks/src/lib.rs @@ -1,6 +1,5 @@ pub mod backend; pub mod compare; -pub mod datasets; mod local_workers; pub mod results; pub mod run; diff --git a/datafusion/distributed-benchmarks/src/main.rs b/datafusion/distributed-benchmarks/src/main.rs index c633cb1f3..c9af510e5 100644 --- a/datafusion/distributed-benchmarks/src/main.rs +++ b/datafusion/distributed-benchmarks/src/main.rs @@ -1,9 +1,7 @@ //! DataFusion Distributed benchmark runner -mod prepare_clickbench; -mod prepare_tpcds; -mod prepare_tpch; use datafusion::error::Result; +use datafusion_datasets::{prepare_clickbench, prepare_tpcds, prepare_tpch}; use datafusion_distributed_benchmarks::{backend::ParquetBenchmarkBackend, compare, run}; use structopt::StructOpt; diff --git a/datafusion/distributed-benchmarks/src/run.rs b/datafusion/distributed-benchmarks/src/run.rs index ba1957e8b..7bc5f1084 100644 --- a/datafusion/distributed-benchmarks/src/run.rs +++ b/datafusion/distributed-benchmarks/src/run.rs @@ -16,7 +16,6 @@ // under the License. use crate::backend::BenchmarkBackend; -use crate::datasets::{clickbench, tpcds, tpch}; use crate::local_workers::LocalWorkers; use crate::results::{BenchResult, BenchmarkRun, QueryIter, dataset_path}; use crate::stats::stats_estimation_q_error; @@ -31,6 +30,7 @@ use datafusion::error::{DataFusionError, Result}; use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion::prelude::*; +use datafusion_datasets::{clickbench, tpcds, tpch}; use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; use datafusion_distributed::test_utils::work_unit_file_scan::{ WorkUnitFileScanCodec, WorkUnitFileScanConfig, WorkUnitFileScanRule, diff --git a/datafusion/distributed-console/Cargo.toml b/datafusion/distributed-console/Cargo.toml index 900ee22b1..5f086d1f1 100644 --- a/datafusion/distributed-console/Cargo.toml +++ b/datafusion/distributed-console/Cargo.toml @@ -19,6 +19,6 @@ tokio-stream = "0.1.18" [dev-dependencies] arrow = "59" async-trait = "0.1.89" -datafusion-distributed-benchmarks = { path = "../distributed-benchmarks" } +datafusion-datasets = { path = "../datasets" } futures = "0.3.31" url = "2.5.7" diff --git a/datafusion/distributed-console/examples/tpcds_runner.rs b/datafusion/distributed-console/examples/tpcds_runner.rs index 22fb676ba..298d26f20 100644 --- a/datafusion/distributed-console/examples/tpcds_runner.rs +++ b/datafusion/distributed-console/examples/tpcds_runner.rs @@ -4,13 +4,13 @@ use datafusion::common::runtime::JoinSet; use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; +use datafusion_datasets::{ + output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpcds, +}; use datafusion_distributed::{ DistributedExt, DistributedMetricsFormat, SessionStateBuilderExt, WorkerResolver, display_plan_ascii, }; -use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpcds, -}; use futures::TryStreamExt; use std::error::Error; use std::num::NonZeroUsize; diff --git a/datafusion/distributed-iceberg-benchmarks/Cargo.toml b/datafusion/distributed-iceberg-benchmarks/Cargo.toml index 0048a859c..a230a23a2 100644 --- a/datafusion/distributed-iceberg-benchmarks/Cargo.toml +++ b/datafusion/distributed-iceberg-benchmarks/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] async-trait = "0.1.89" datafusion = { workspace = true, features = ["parquet", "sql"] } +datafusion-datasets = { path = "../datasets" } datafusion-distributed-benchmarks = { path = "../distributed-benchmarks" } datafusion-distributed-iceberg = { path = "../distributed-iceberg" } env_logger = "0.11.8" diff --git a/datafusion/distributed-iceberg-benchmarks/src/dataset.rs b/datafusion/distributed-iceberg-benchmarks/src/dataset.rs index 5ce03d59b..290811030 100644 --- a/datafusion/distributed-iceberg-benchmarks/src/dataset.rs +++ b/datafusion/distributed-iceberg-benchmarks/src/dataset.rs @@ -9,7 +9,7 @@ use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err}; use datafusion::execution::context::SessionContext; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::ParquetReadOptions; -use datafusion_distributed_benchmarks::datasets::output::DatasetOutput; +use datafusion_datasets::output::DatasetOutput; use datafusion_distributed_iceberg::IcebergIntegrationOptions; use datafusion_distributed_iceberg::iceberg; use futures::StreamExt; diff --git a/datafusion/distributed-iceberg-benchmarks/src/prepare.rs b/datafusion/distributed-iceberg-benchmarks/src/prepare.rs index c7ee26b62..676c173c8 100644 --- a/datafusion/distributed-iceberg-benchmarks/src/prepare.rs +++ b/datafusion/distributed-iceberg-benchmarks/src/prepare.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use super::dataset::{convert_parquet_to_iceberg, output_path}; use datafusion::common::{Result, exec_datafusion_err}; -use datafusion_distributed_benchmarks::datasets::output::DatasetOutput; +use datafusion_datasets::output::DatasetOutput; use structopt::StructOpt; /// Convert local Parquet tables into an Iceberg dataset. diff --git a/datafusion/distributed/Cargo.toml b/datafusion/distributed/Cargo.toml index 6081907ea..a7d19da07 100644 --- a/datafusion/distributed/Cargo.toml +++ b/datafusion/distributed/Cargo.toml @@ -123,7 +123,7 @@ chaos-tests = [] sysinfo = ["dep:sysinfo"] [dev-dependencies] -datafusion-distributed-benchmarks = { path = "../distributed-benchmarks" } +datafusion-datasets = { path = "../datasets" } # The crate's own tests drive `SessionContext::sql` and register parquet tables, so they need # both features regardless of which ones a consumer selects. Requesting them here keeps # `--no-default-features` test runs compiling without forcing either on library consumers. diff --git a/datafusion/distributed/tests/chaos.rs b/datafusion/distributed/tests/chaos.rs index 850a6a952..c00df0bb2 100644 --- a/datafusion/distributed/tests/chaos.rs +++ b/datafusion/distributed/tests/chaos.rs @@ -6,13 +6,13 @@ mod tests { use datafusion::error::DataFusionError; use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::prelude::SessionContext; + use datafusion_datasets::{ + output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, + }; use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; use datafusion_distributed::{ DistributedConfig, DistributedExt, SessionStateBuilderExt, Worker, WorkerQueryContext, }; - use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, - }; use moka::future::FutureExt; use rand::SeedableRng; use rand::prelude::{Rng, StdRng}; diff --git a/datafusion/distributed/tests/clickbench_correctness_test.rs b/datafusion/distributed/tests/clickbench_correctness_test.rs index 39a7580c8..a5a6b3573 100644 --- a/datafusion/distributed/tests/clickbench_correctness_test.rs +++ b/datafusion/distributed/tests/clickbench_correctness_test.rs @@ -5,6 +5,7 @@ mod tests { use datafusion::error::Result; use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion::prelude::SessionContext; + use datafusion_datasets::{clickbench, output::TESTDATA_ROOT, register_tables}; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::test_utils::property_based::{ compare_ordering, compare_result_set, @@ -12,9 +13,6 @@ mod tests { use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - clickbench, output::TESTDATA_ROOT, register_tables, - }; use std::ops::Range; use std::sync::Arc; use tokio::sync::OnceCell; diff --git a/datafusion/distributed/tests/clickbench_plans_test.rs b/datafusion/distributed/tests/clickbench_plans_test.rs index e3087515e..394618337 100644 --- a/datafusion/distributed/tests/clickbench_plans_test.rs +++ b/datafusion/distributed/tests/clickbench_plans_test.rs @@ -1,13 +1,11 @@ #[cfg(all(feature = "integration", feature = "clickbench", test))] mod tests { use datafusion::error::Result; + use datafusion_datasets::{clickbench, output::TESTDATA_ROOT, register_tables}; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - clickbench, output::TESTDATA_ROOT, register_tables, - }; use std::ops::Range; use tokio::sync::OnceCell; diff --git a/datafusion/distributed/tests/shuffle.rs b/datafusion/distributed/tests/shuffle.rs index 893d8db0a..d8988e1de 100644 --- a/datafusion/distributed/tests/shuffle.rs +++ b/datafusion/distributed/tests/shuffle.rs @@ -176,13 +176,13 @@ mod tests { mod tpch_two_phase_tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; + use datafusion_datasets::{ + output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, + }; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExt, assert_snapshot, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, - }; use futures::TryStreamExt; use std::error::Error; use std::fmt::Display; diff --git a/datafusion/distributed/tests/stateful_data_cleanup.rs b/datafusion/distributed/tests/stateful_data_cleanup.rs index c6766888f..cd501fd77 100644 --- a/datafusion/distributed/tests/stateful_data_cleanup.rs +++ b/datafusion/distributed/tests/stateful_data_cleanup.rs @@ -3,11 +3,11 @@ mod tests { use datafusion::common::{Result, instant::Instant}; use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::{DefaultSessionBuilder, DistributedExt, Worker}; - use datafusion_distributed_benchmarks::datasets::{ + use datafusion_datasets::{ output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, }; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::{DefaultSessionBuilder, DistributedExt, Worker}; use futures::TryStreamExt; use std::time::Duration; use test_case::test_case; diff --git a/datafusion/distributed/tests/tpcds_correctness_test.rs b/datafusion/distributed/tests/tpcds_correctness_test.rs index 922f27c86..e428130bd 100644 --- a/datafusion/distributed/tests/tpcds_correctness_test.rs +++ b/datafusion/distributed/tests/tpcds_correctness_test.rs @@ -5,6 +5,11 @@ mod tests { use datafusion::error::Result; use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion::prelude::SessionContext; + use datafusion_datasets::{ + output::TESTDATA_ROOT, + parquet_layout::{Dictionary, ParquetLayout}, + register_tables, tpcds, + }; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::test_utils::property_based::{ compare_ordering, compare_result_set, @@ -12,11 +17,6 @@ mod tests { use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, - parquet_layout::{Dictionary, ParquetLayout}, - register_tables, tpcds, - }; use std::sync::Arc; use tokio::sync::OnceCell; diff --git a/datafusion/distributed/tests/tpcds_plans_test.rs b/datafusion/distributed/tests/tpcds_plans_test.rs index e540d97ee..5caffad4b 100644 --- a/datafusion/distributed/tests/tpcds_plans_test.rs +++ b/datafusion/distributed/tests/tpcds_plans_test.rs @@ -1,15 +1,15 @@ #[cfg(all(feature = "integration", feature = "tpcds", test))] mod tests { use datafusion::error::Result; - use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; - use datafusion_distributed::{ - DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, - }; - use datafusion_distributed_benchmarks::datasets::{ + use datafusion_datasets::{ output::TESTDATA_ROOT, parquet_layout::{Dictionary, ParquetLayout}, register_tables, tpcds, }; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, + }; use tokio::sync::OnceCell; const NUM_WORKERS: usize = 4; diff --git a/datafusion/distributed/tests/tpch_correctness_test.rs b/datafusion/distributed/tests/tpch_correctness_test.rs index 34bed6ecb..8d8f7c3f0 100644 --- a/datafusion/distributed/tests/tpch_correctness_test.rs +++ b/datafusion/distributed/tests/tpch_correctness_test.rs @@ -2,11 +2,11 @@ mod tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; - use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; - use datafusion_distributed_benchmarks::datasets::{ + use datafusion_datasets::{ output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, }; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; use futures::TryStreamExt; use std::error::Error; use std::fmt::Display; diff --git a/datafusion/distributed/tests/tpch_plans_test.rs b/datafusion/distributed/tests/tpch_plans_test.rs index 34a373b99..64ac185c5 100644 --- a/datafusion/distributed/tests/tpch_plans_test.rs +++ b/datafusion/distributed/tests/tpch_plans_test.rs @@ -1,12 +1,12 @@ #[cfg(all(feature = "integration", feature = "tpch", test))] mod tests { + use datafusion_datasets::{ + output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, + }; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExt, assert_snapshot, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, - }; use std::error::Error; use tokio::sync::OnceCell; diff --git a/datafusion/distributed/tests/tpch_sorted_correctness_test.rs b/datafusion/distributed/tests/tpch_sorted_correctness_test.rs index d3b22d0a1..501025cca 100644 --- a/datafusion/distributed/tests/tpch_sorted_correctness_test.rs +++ b/datafusion/distributed/tests/tpch_sorted_correctness_test.rs @@ -2,11 +2,11 @@ mod tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; - use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; - use datafusion_distributed_benchmarks::datasets::{ + use datafusion_datasets::{ output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, }; + use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; + use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; use futures::TryStreamExt; use std::error::Error; use std::fmt::Display; diff --git a/datafusion/distributed/tests/tpch_sorted_plans_test.rs b/datafusion/distributed/tests/tpch_sorted_plans_test.rs index 46a512d1a..fafb9f499 100644 --- a/datafusion/distributed/tests/tpch_sorted_plans_test.rs +++ b/datafusion/distributed/tests/tpch_sorted_plans_test.rs @@ -1,12 +1,12 @@ #[cfg(all(feature = "integration", feature = "tpch", test))] mod tests { + use datafusion_datasets::{ + output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, + }; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExt, assert_snapshot, display_plan_ascii, }; - use datafusion_distributed_benchmarks::datasets::{ - output::TESTDATA_ROOT, parquet_layout::ParquetLayout, register_tables, tpch, - }; use std::error::Error; use tokio::sync::OnceCell; From fa4a1bb5344bbeb32358c3e53d43913d23b393e7 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:34:36 +0100 Subject: [PATCH 02/31] refactor: compile all integration tests into a single test binary (#868) Compiles all integration tests of `datafusion-distributed` into one test binary instead of one binary per file. Before this change, each of the 32 files in `tests/` (plus `dynamic_filtering/`) produced its own ~300 MB binary that linked the whole dependency tree. After a library edit, cargo relinked every one of them. Running a single file's tests locally: `cargo test --features integration --test integration udfs::`. Rebuilding the library plus all test targets after a one-line library edit went from 15-16s to 4-8s locally --- .github/workflows/ci.yml | 18 +++++----- datafusion/distributed/Cargo.toml | 7 ++-- .../tests/dynamic_filtering/aggregates.rs | 2 +- .../dynamic_filtering/collect_left_join.rs | 2 +- .../tests/dynamic_filtering/config.rs | 2 +- .../dynamic_filtering/{main.rs => mod.rs} | 0 .../dynamic_filtering/partitioned_join.rs | 2 +- .../tests/dynamic_filtering/sorts.rs | 2 +- datafusion/distributed/tests/main.rs | 36 +++++++++++++++++++ docs/source/contributor-guide/03-tests.md | 3 ++ 10 files changed, 58 insertions(+), 16 deletions(-) rename datafusion/distributed/tests/dynamic_filtering/{main.rs => mod.rs} (100%) create mode 100644 datafusion/distributed/tests/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5463ef4a5..a2cab4487 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test tpch_correctness_test + - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_correctness_test:: env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -62,7 +62,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test tpch_sorted_correctness_test + - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_correctness_test:: env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -71,14 +71,14 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test tpch_plans_test + - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_plans_test:: tpch-sorted-plans-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test tpch_sorted_plans_test + - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_plans_test:: tpcds-correctness-test: runs-on: ubuntu-latest @@ -90,7 +90,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }} + - run: cargo test -p datafusion-distributed --features tpcds --test integration tpcds_correctness_test::tests::test_tpcds_shard${{ matrix.shard }} env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -99,7 +99,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpcds --test tpcds_plans_test + - run: cargo test -p datafusion-distributed --features tpcds --test integration tpcds_plans_test:: clickbench-correctness-test: runs-on: ubuntu-latest @@ -114,7 +114,7 @@ jobs: with: path: testdata/clickbench_parquet/ key: "clickbench_parquet" - - run: cargo test -p datafusion-distributed --features clickbench --test clickbench_correctness_test + - run: cargo test -p datafusion-distributed --features clickbench --test integration clickbench_correctness_test:: env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -127,7 +127,7 @@ jobs: with: path: testdata/clickbench_parquet/ key: "clickbench_parquet" - - run: cargo test -p datafusion-distributed --features clickbench --test clickbench_plans_test + - run: cargo test -p datafusion-distributed --features clickbench --test integration clickbench_plans_test:: iceberg-test: runs-on: ubuntu-latest @@ -145,7 +145,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch,chaos-tests --test chaos + - run: cargo test -p datafusion-distributed --features tpch,chaos-tests --test integration chaos:: env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} diff --git a/datafusion/distributed/Cargo.toml b/datafusion/distributed/Cargo.toml index a7d19da07..3a999f80d 100644 --- a/datafusion/distributed/Cargo.toml +++ b/datafusion/distributed/Cargo.toml @@ -7,10 +7,13 @@ license = "Apache-2.0" documentation = "https://datafusion-contrib.github.io/datafusion-distributed/" repository = "https://github.com/datafusion-contrib/datafusion-distributed" readme = "../../README.md" +autotests = false +# Every integration test is a module of `tests/main.rs`, so they all link into one binary +# instead of one binary per file. [[test]] -name = "dynamic_filtering" -path = "tests/dynamic_filtering/main.rs" +name = "integration" +path = "tests/main.rs" required-features = ["integration"] # The examples live at the repository root (`examples/`), outside this package directory. diff --git a/datafusion/distributed/tests/dynamic_filtering/aggregates.rs b/datafusion/distributed/tests/dynamic_filtering/aggregates.rs index e6b6464fb..c58274e8d 100644 --- a/datafusion/distributed/tests/dynamic_filtering/aggregates.rs +++ b/datafusion/distributed/tests/dynamic_filtering/aggregates.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::TestQuery; + use crate::dynamic_filtering::common::TestQuery; use datafusion::common::Result; use datafusion_distributed::assert_snapshot; diff --git a/datafusion/distributed/tests/dynamic_filtering/collect_left_join.rs b/datafusion/distributed/tests/dynamic_filtering/collect_left_join.rs index 184629239..9b67bca83 100644 --- a/datafusion/distributed/tests/dynamic_filtering/collect_left_join.rs +++ b/datafusion/distributed/tests/dynamic_filtering/collect_left_join.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::TestQuery; + use crate::dynamic_filtering::common::TestQuery; use datafusion::common::Result; use datafusion_distributed::assert_snapshot; diff --git a/datafusion/distributed/tests/dynamic_filtering/config.rs b/datafusion/distributed/tests/dynamic_filtering/config.rs index 098bab5d1..c87716f97 100644 --- a/datafusion/distributed/tests/dynamic_filtering/config.rs +++ b/datafusion/distributed/tests/dynamic_filtering/config.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::TestQuery; + use crate::dynamic_filtering::common::TestQuery; use datafusion::common::Result; use datafusion_distributed::assert_snapshot; diff --git a/datafusion/distributed/tests/dynamic_filtering/main.rs b/datafusion/distributed/tests/dynamic_filtering/mod.rs similarity index 100% rename from datafusion/distributed/tests/dynamic_filtering/main.rs rename to datafusion/distributed/tests/dynamic_filtering/mod.rs diff --git a/datafusion/distributed/tests/dynamic_filtering/partitioned_join.rs b/datafusion/distributed/tests/dynamic_filtering/partitioned_join.rs index e8eb3ced2..cc6523fe1 100644 --- a/datafusion/distributed/tests/dynamic_filtering/partitioned_join.rs +++ b/datafusion/distributed/tests/dynamic_filtering/partitioned_join.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::{TestQuery, execute_range_partitioned_query}; + use crate::dynamic_filtering::common::{TestQuery, execute_range_partitioned_query}; use datafusion::common::Result; use datafusion_distributed::assert_snapshot; use datafusion_distributed::test_utils::insta::insta::allow_duplicates; diff --git a/datafusion/distributed/tests/dynamic_filtering/sorts.rs b/datafusion/distributed/tests/dynamic_filtering/sorts.rs index e3e2254c5..c13068a31 100644 --- a/datafusion/distributed/tests/dynamic_filtering/sorts.rs +++ b/datafusion/distributed/tests/dynamic_filtering/sorts.rs @@ -1,6 +1,6 @@ #[cfg(test)] mod tests { - use crate::common::TestQuery; + use crate::dynamic_filtering::common::TestQuery; use datafusion::common::Result; use datafusion_distributed::assert_snapshot; diff --git a/datafusion/distributed/tests/main.rs b/datafusion/distributed/tests/main.rs new file mode 100644 index 000000000..bfe99619a --- /dev/null +++ b/datafusion/distributed/tests/main.rs @@ -0,0 +1,36 @@ +//! All integration tests compile into this single test binary. Every file in `tests/` is a +//! module here, so adding a new test file requires adding a `mod` line below. + +mod chaos; +mod clickbench_correctness_test; +mod clickbench_plans_test; +mod custom_config_extension; +mod custom_extension_codec; +mod distributed_aggregation; +mod distributed_unions; +mod dynamic_filtering; +mod empty_build_partitions; +mod empty_columns_between_workers; +mod error_propagation; +mod explain_analyze; +mod introspection; +mod join; +mod local_connections; +mod metrics_collection; +mod multi_task_collect_join_repros; +mod passthrough_headers; +mod shuffle; +mod sorted_data_source; +mod stateful_data_cleanup; +mod stateful_execution_plan; +mod task_estimator_test; +mod task_finished_handler; +mod tpcds_correctness_test; +mod tpcds_plans_test; +mod tpch_correctness_test; +mod tpch_plans_test; +mod tpch_sorted_correctness_test; +mod tpch_sorted_plans_test; +mod udfs; +mod work_unit_feed; +mod worker_plan_hook; diff --git a/docs/source/contributor-guide/03-tests.md b/docs/source/contributor-guide/03-tests.md index bf565c5df..e83e0ac78 100644 --- a/docs/source/contributor-guide/03-tests.md +++ b/docs/source/contributor-guide/03-tests.md @@ -79,6 +79,9 @@ Integration tests are slower but cover a wide range of functionality. ```bash # Run unit and integration tests cargo test --features integration + +# Run the integration tests of one file, e.g. tests/udfs.rs +cargo test --features integration --test integration udfs:: ``` ## Running Benchmark Tests From 33a15eb041c8f8dc19b675feef6ce17bec57a4da Mon Sep 17 00:00:00 2001 From: Joy Bestourous <32601978+joybestourous@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:39:21 -0500 Subject: [PATCH 03/31] refactor: name the adaptive network boundary builder (#872) AQE stage preparation currently lives in a large async closure, which makes subsequent startup changes harder to review. Move that callback into `DynamicPlanBuilder`, implementing the existing `NetworkBoundaryBuilder` trait. Task setup still joins all destinations before sampling kickoff; coalesce behavior, task counts, dynamic filters, statistics and metrics are unchanged. This is a preliminary refactor for #852. Validation: 343 library tests passed, one ignored; formatting passed. The moved callback body was checked for equivalence after accounting for borrowed fields and the removed closure wrappers. Co-authored-by: Codex GPT-6 --- .../src/coordinator/prepare_dynamic_plan.rs | 247 +++++++++--------- .../src/distributed_planner/mod.rs | 3 +- 2 files changed, 132 insertions(+), 118 deletions(-) diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index e8453d4d9..f29fa8d01 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -2,8 +2,8 @@ use crate::common::{TreeNodeExt, element_wise_sum, vec_avg_reduce, vec_div, vec_ use crate::coordinator::distributed::PreparedPlan; use crate::coordinator::query_coordinator::QueryCoordinator; use crate::distributed_planner::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, ProducerHead, calculate_cost, - inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + ProducerHead, calculate_cost, inject_network_boundaries, }; use crate::dynamic_filtering::{ is_remote_dynamic_filtering_enabled, orphan_dynamic_filter_consumers, @@ -14,6 +14,7 @@ use crate::{ BytesCounterMetric, CoordinatorToWorkerMsg, LoadInfo, MaxGaugeMetric, NetworkBoundaryExt, NetworkCoalesceExec, Stage, TaskCountAnnotation, }; +use async_trait::async_trait; use dashmap::DashMap; use datafusion::common::stats::Precision; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; @@ -33,124 +34,13 @@ pub(super) async fn prepare_dynamic_plan( query_coordinator: &Arc, base_plan: &Arc, ) -> Result { - let plans_for_viz = Arc::new(PlanReconstructor::default()); + let plans_for_viz = PlanReconstructor::default(); let head_stage = inject_network_boundaries( Arc::clone(base_plan), - |mut input_stage: LocalStage, nb_type: TypeId, nb_ctx: &InjectNetworkBoundaryContext| { - let mut metrics = MetricsSet::new(); - - // At this point, input_stage.plan has two kind of leaf nodes: - // - The ones that naturally do not read from any children, like DataSourceExec - // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration - // of this same function. - // Both types of leaf nodes contain very valuable and accurate statistics that are used - // here for computing an estimation of the compute cost (measured in bytes): - // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their - // data source, like parquet files. - // - Network boundaries contain statistics collected from runtime information, gathered - // by the SamplerExec injected by this same function. - let cost = calculate_cost(&input_stage.plan)?; - metrics.push(BytesCounterMetric::new_metric( - "cpu_cost", - *cost.cpu.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "memory_cost", - *cost.memory.get_value().unwrap_or(&0), - )); - metrics.push(BytesCounterMetric::new_metric( - "network_cost", - *cost.network.get_value().unwrap_or(&0), - )); - let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 - / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 - / input_stage - .plan - .output_partitioning() - .partition_count() - .max(1) as f64; - let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); - let task_count = nb_ctx - .task_count(&input_stage.plan)? - .merge(TaskCountAnnotation::soft(compute_based_task_count))?; - - // Propagate the final task_count inferred based on runtime statistics and compute cost. - // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the - // plan is finally left ready for distribution. - input_stage.plan = nb_ctx - .propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; - input_stage.tasks = task_count.as_usize(); - // In order to infer the compute the cost of the stage above this one, here a sampler - // is injected to gather runtime statistics. - input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; - let dynamic_filter_anchors = - if is_remote_dynamic_filtering_enabled(query_coordinator.session_config())? { - orphan_dynamic_filter_consumers(&input_stage.plan)? - } else { - vec![] - }; - - let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); - - let query_coordinator = Arc::clone(query_coordinator); - let plans_for_viz = Arc::clone(&plans_for_viz); - - Ok(async move { - let mut stage_coordinator = query_coordinator.stage_coordinator(&input_stage); - - let mut futures = Vec::with_capacity(input_stage.tasks); - for task_i in 0..input_stage.tasks { - futures.push(stage_coordinator.init_bidirectional_stream(task_i)); - } - let results = futures::future::try_join_all(futures).await?; - - let mut workers = Vec::with_capacity(input_stage.tasks); - for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() - { - workers.push(url); - load_info_rxs.push({ - let rx = - stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); - UnboundedReceiverStream::new(rx) - }); - let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); - stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; - } - stage_coordinator.seal_dynamic_filter_stage()?; - - let (stats, consumer_tc) = if nb_type == TypeId::of::() { - (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) - } else { - let (stats, new_metrics) = - gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; - metrics.extend(new_metrics); - // returning Desired(0) here is our way to tell the planner that we don't care - // about the task count assigned to the network boundary in the consumer stage, - // and we don't want it to affect other task count decisions. A 0.0 hint does - // not mask smaller fractional values via max-merge, and does not inflate - // UNION sums when this boundary sits under a child of a - // ChildrenIsolatorUnionExec. - (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) - }; - - // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan - // before it's moved: the returned stage is remote and carries no plan to read it - // back from. - let input_properties = Arc::clone(input_stage.plan.properties()); - plans_for_viz.insert(input_stage.num, input_stage.plan, metrics); - Ok(NetworkBoundaryBuilderResult { - consumer_task_count: consumer_tc, - input_stage: Stage::Remote(RemoteStage { - query_id: input_stage.query_id, - num: input_stage.num, - workers, - runtime_stats: stats, - dynamic_filter_anchors, - }), - input_properties, - }) - }) + DynamicPlanBuilder { + query_coordinator, + plans_for_viz: &plans_for_viz, }, query_coordinator.session_config(), ) @@ -162,6 +52,129 @@ pub(super) async fn prepare_dynamic_plan( }) } +struct DynamicPlanBuilder<'a> { + query_coordinator: &'a Arc, + plans_for_viz: &'a PlanReconstructor, +} + +#[async_trait] +impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> { + async fn build<'a>( + &'a self, + mut input_stage: LocalStage, + nb_type: TypeId, + nb_ctx: &'a InjectNetworkBoundaryContext<'a>, + ) -> Result { + let mut metrics = MetricsSet::new(); + + // At this point, input_stage.plan has two kind of leaf nodes: + // - The ones that naturally do not read from any children, like DataSourceExec + // - Network boundaries whose Stage was set to Stage::Remote by a previous iteration + // of this same function. + // Both types of leaf nodes contain very valuable and accurate statistics that are used + // here for computing an estimation of the compute cost (measured in bytes): + // - DataSourceExec (or natural leaf nodes) contain stats pulled directly from their + // data source, like parquet files. + // - Network boundaries contain statistics collected from runtime information, gathered + // by the SamplerExec injected by this same function. + let cost = calculate_cost(&input_stage.plan)?; + metrics.push(BytesCounterMetric::new_metric( + "cpu_cost", + *cost.cpu.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "memory_cost", + *cost.memory.get_value().unwrap_or(&0), + )); + metrics.push(BytesCounterMetric::new_metric( + "network_cost", + *cost.network.get_value().unwrap_or(&0), + )); + let compute_based_task_count = *cost.cpu.get_value().unwrap_or(&0) as f64 + / nb_ctx.d_cfg.dynamic_bytes_per_partition.max(1) as f64 + / input_stage + .plan + .output_partitioning() + .partition_count() + .max(1) as f64; + let compute_based_task_count = compute_based_task_count.min(nb_ctx.max_tasks()? as f64); + let task_count = nb_ctx + .task_count(&input_stage.plan)? + .merge(TaskCountAnnotation::soft(compute_based_task_count))?; + + // Propagate the final task_count inferred based on runtime statistics and compute cost. + // Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the + // plan is finally left ready for distribution. + input_stage.plan = + nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?; + input_stage.tasks = task_count.as_usize(); + // In order to infer the compute the cost of the stage above this one, here a sampler + // is injected to gather runtime statistics. + input_stage.plan = ProducerHead::insert_sampler(input_stage.plan)?; + let dynamic_filter_anchors = + if is_remote_dynamic_filtering_enabled(self.query_coordinator.session_config())? { + orphan_dynamic_filter_consumers(&input_stage.plan)? + } else { + vec![] + }; + + let mut load_info_rxs = Vec::with_capacity(input_stage.tasks); + + let mut stage_coordinator = self.query_coordinator.stage_coordinator(&input_stage); + + let mut futures = Vec::with_capacity(input_stage.tasks); + for task_i in 0..input_stage.tasks { + futures.push(stage_coordinator.init_bidirectional_stream(task_i)); + } + let results = futures::future::try_join_all(futures).await?; + + let mut workers = Vec::with_capacity(input_stage.tasks); + for (task_i, (url, worker_tx, worker_rx_stream)) in results.into_iter().enumerate() { + workers.push(url); + load_info_rxs.push({ + let rx = stage_coordinator.worker_to_coordinator_task(task_i, worker_rx_stream); + UnboundedReceiverStream::new(rx) + }); + let _ = worker_tx.send(CoordinatorToWorkerMsg::KickOffSampling); + stage_coordinator.coordinator_to_worker_task(task_i, worker_tx)?; + } + stage_coordinator.seal_dynamic_filter_stage()?; + + let (stats, consumer_tc) = if nb_type == TypeId::of::() { + (None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0)) + } else { + let (stats, new_metrics) = + gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?; + metrics.extend(new_metrics); + // returning Desired(0) here is our way to tell the planner that we don't care + // about the task count assigned to the network boundary in the consumer stage, + // and we don't want it to affect other task count decisions. A 0.0 hint does + // not mask smaller fractional values via max-merge, and does not inflate + // UNION sums when this boundary sits under a child of a + // ChildrenIsolatorUnionExec. + (Some(Arc::new(stats)), TaskCountAnnotation::soft(0.0)) + }; + + // Capture the output partitioning of the (rescaled, sampler-wrapped) input plan + // before it's moved: the returned stage is remote and carries no plan to read it + // back from. + let input_properties = Arc::clone(input_stage.plan.properties()); + self.plans_for_viz + .insert(input_stage.num, input_stage.plan, metrics); + Ok(NetworkBoundaryBuilderResult { + consumer_task_count: consumer_tc, + input_stage: Stage::Remote(RemoteStage { + query_id: input_stage.query_id, + num: input_stage.num, + workers, + runtime_stats: stats, + dynamic_filter_anchors, + }), + input_properties, + }) + } +} + /// Reconstructs the plan dynamically as stages get transitioned to Remote and get sent to the /// respective workers. /// diff --git a/datafusion/distributed/src/distributed_planner/mod.rs b/datafusion/distributed/src/distributed_planner/mod.rs index b49645214..756ab456f 100644 --- a/datafusion/distributed/src/distributed_planner/mod.rs +++ b/datafusion/distributed/src/distributed_planner/mod.rs @@ -14,7 +14,8 @@ mod statistics; pub use distributed_config::DistributedConfig; pub use distributed_query_planner::create_distributed_plan; pub(crate) use inject_network_boundaries::{ - InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, inject_network_boundaries, + InjectNetworkBoundaryContext, NetworkBoundaryBuilder, NetworkBoundaryBuilderResult, + inject_network_boundaries, }; pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt, ProducerHead}; pub use session_state_builder_ext::SessionStateBuilderExt; From cb6c9c437d5886cce2dde545e4216dec220b75eb Mon Sep 17 00:00:00 2001 From: Joy Bestourous <32601978+joybestourous@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:54:28 -0500 Subject: [PATCH 04/31] perf: benchmark task startup overlap (#873) Task setup skew can postpone useful work on a ready worker. Add a Criterion benchmark for that scheduling case before the execution changes in #852. This PR changes no library behavior and adds no dependencies. The benchmark runs one CPU-heavy task and one light task over localhost gRPC, in both static and AQE modes. Each mode has three cases: no routing delay, a 250 ms routing delay on the light task, and the same delay on the heavy task. It measures execution through complete output collection, excluding worker startup and planning. Every iteration checks output `[0, 1]` and exactly one heavy remote execution. An untimed preflight reports whether CPU work starts and finishes before delayed sibling routing returns. Applied unchanged to #852, the benchmark demonstrates the conditional benefit: | Mode | Case | Baseline | With #852 | Paired change | |---|---|---:|---:|---:| | Static | No delay | 228.15 ms | 227.84 ms | -0.14% | | Static | Delayed light task | 479.29 ms | 294.10 ms | **-38.64%** | | Static | Delayed heavy task | 479.09 ms | 478.60 ms | -0.10% | | AQE | No delay | 228.23 ms | 227.86 ms | -0.16% | | AQE | Delayed light task | 479.37 ms | 294.36 ms | **-38.60%** | | AQE | Delayed heavy task | 479.52 ms | 478.79 ms | -0.15% | In both delayed-light preflights, heavy work started and finished before sibling routing returned with #852; neither witness occurred on the baseline. The controls stayed essentially flat. This demonstrates value when ready useful work can overlap a sibling's setup delay; it does not predict representative S3 or multi-host speedups. Validation: optimized builds and all six cases passed on baseline and candidate. Formatting and Criterion-target Clippy with warnings denied passed.
Measurement scope and reproduction Measurements used baseline `0291e08` (main `e5d6ea3` plus this benchmark) and candidate source matching #852 at `354d386`. The benchmark and lockfile were identical. The candidate binary was built before the final commit; source hashes were verified against that commit. This was a controlled synthetic experiment on a shared host, with the same four-CPU affinity for both binaries, four Tokio threads, ten Criterion samples per case, one-second warmup and a five-second target measurement window. Three outer pairs were attempted in baseline/candidate, candidate/baseline, baseline/candidate order. Two opposite-order pairs passed the predefined resource gate. The entire third pair was excluded because baseline selected-CPU average utilization p95 was 85.31%, exceeding the 85% threshold; the maximum-utilization threshold was 98%. Raw results for both sides of the excluded pair were retained. Results were not filtered by measured latency. Times above are medians of the accepted per-run Criterion medians; changes are medians of the paired percentage changes. Two outer pairs illustrate this scheduling case, rather than establishing production performance or precise population estimates. ```shell cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --save-baseline main # Run the same benchmark on the candidate revision: cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --baseline main ``` Use separate build directories or rebuild both local packages when switching revisions. Confirm the candidate's delayed-light preflight shows early work and the baseline's does not before interpreting timings.
--------- Co-authored-by: Codex GPT-6 --- datafusion/distributed-benchmarks/Cargo.toml | 4 + .../benches/startup_overlap.rs | 229 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 datafusion/distributed-benchmarks/benches/startup_overlap.rs diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index 866b650c6..5245bb1ff 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -57,3 +57,7 @@ harness = false [[bench]] name = "metrics_collection" harness = false + +[[bench]] +name = "startup_overlap" +harness = false diff --git a/datafusion/distributed-benchmarks/benches/startup_overlap.rs b/datafusion/distributed-benchmarks/benches/startup_overlap.rs new file mode 100644 index 000000000..f171d7189 --- /dev/null +++ b/datafusion/distributed-benchmarks/benches/startup_overlap.rs @@ -0,0 +1,229 @@ +//! # Task startup overlap +//! +//! ```shell +//! cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --save-baseline main +//! # Run the same benchmark on the candidate revision: +//! cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --baseline main +//! ``` +//! +//! This controlled benchmark runs two tasks over localhost gRPC in static and AQE +//! mode. Task 0 performs 100 million dependent integer operations; task 1 returns +//! one row. The existing localhost fixture starts three endpoints so both active +//! tasks can route away from the coordinator's local endpoint. Each query must +//! return task indices `[0, 1]`, and the CPU function must execute remotely once. +//! +//! `delay_light_task` adds 250 ms to task 1's routing setup, allowing task 0's CPU +//! work to overlap that wait. `delay_heavy_task` delays task 0 instead, keeping the +//! wait on the critical path; `no_delay` provides the other control. The delay is +//! only in routing setup, not in the CPU workload. Criterion measures execution +//! through complete output collection; worker startup and query planning are +//! excluded. An untimed preflight also reports first-batch time and whether remote +//! CPU work started and finished before the delayed task returned a ready +//! destination. The witnesses are reported without requiring overlap, so the +//! benchmark also runs before the optimization. +//! +//! Run both revisions on a quiet host and alternate comparisons. Verify a positive +//! overlap witness in the candidate's `delay_light_task` preflight and its absence +//! in the baseline before interpreting timings. This isolates scheduling value +//! under controlled setup skew; it does not replace representative suite results. + +use arrow::array::Int64Array; +use arrow::datatypes::DataType; +use async_trait::async_trait; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion::common::{Result, exec_err}; +use datafusion::execution::SessionState; +use datafusion::logical_expr::{ScalarUDF, Volatility, create_udf}; +use datafusion::physical_plan::execute_stream; +use datafusion::prelude::SessionContext; +use datafusion_distributed::test_utils::localhost::start_localhost_context; +use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, + url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, +}; +use datafusion_distributed::{ + DistributedExt, MappedWorkerSessionBuilderExt, RouteTaskEvent, RouteTaskEventResponse, + RouteTaskHandler, WorkerQueryContext, +}; +use futures::TryStreamExt; +use std::hint::black_box; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::time::timeout; + +const QUERY: &str = "SELECT cpu_work(task_index) FROM url_emitter(2, 2, 'startup')"; +const SETUP_DELAY: Duration = Duration::from_millis(250); +const WAIT: Duration = Duration::from_secs(10); + +fn startup_overlap(c: &mut Criterion) { + let rt = RuntimeBuilder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("startup_overlap"); + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(5)); + + for (mode, adaptive) in [("static", false), ("aqe", true)] { + for (scenario, delayed_task) in [ + ("no_delay", None), + ("delay_light_task", Some(1)), + ("delay_heavy_task", Some(0)), + ] { + let probe = Arc::new(Probe::default()); + let udf = Arc::new(cpu_work(Some(Arc::clone(&probe)))); + let worker = build_state.map(move |builder| { + Ok(builder + .with_scalar_functions(vec![Arc::clone(&udf)]) + .build()) + }); + // Routing uses URLs 2 and 1, avoiding URL 0's local-worker shortcut. + let (mut ctx, _workers, _) = rt.block_on(start_localhost_context(3, worker)); + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + ctx.register_udf(cpu_work(None)); + let ctx = ctx + .with_distributed_dynamic_task_count(adaptive) + .unwrap() + .with_distributed_route_task_handler(SetupDelay { + delayed_task, + probe: Arc::clone(&probe), + }); + rt.block_on(ctx.sql("SET datafusion.execution.target_partitions = 2")) + .unwrap(); + + let (first, total) = rt.block_on(run_query(&ctx, &probe)).unwrap(); + println!( + "preflight {mode}/{scenario}: first={first:?} total={total:?} \ + heavy_started_before_ready={} heavy_finished_before_ready={}", + probe.started_before_ready.load(Ordering::SeqCst), + probe.finished_before_ready.load(Ordering::SeqCst), + ); + group.bench_function(BenchmarkId::new(mode, scenario), |b| { + b.iter_custom(|iterations| { + rt.block_on(async { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + elapsed += run_query(&ctx, &probe).await.unwrap().1; + } + elapsed + }) + }) + }); + } + } + group.finish(); +} + +async fn run_query(ctx: &SessionContext, probe: &Probe) -> Result<(Duration, Duration)> { + let plan = ctx.sql(QUERY).await?.create_physical_plan().await?; + probe.delayed_ready.store(false, Ordering::SeqCst); + probe.started_before_ready.store(false, Ordering::SeqCst); + probe.finished_before_ready.store(false, Ordering::SeqCst); + probe.heavy_executions.store(0, Ordering::SeqCst); + let start = Instant::now(); + let mut output = execute_stream(plan, ctx.task_ctx())?; + let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); + let first_elapsed = start.elapsed(); + let mut batches = vec![first]; + batches.extend( + timeout(WAIT, output.try_collect::>()) + .await + .unwrap()?, + ); + let total_elapsed = start.elapsed(); + let mut values: Vec<_> = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + values.sort_unstable(); + assert_eq!(values, vec![0, 1]); + assert_eq!(probe.heavy_executions.load(Ordering::SeqCst), 1); + Ok((first_elapsed, total_elapsed)) +} + +async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .build()) +} + +#[derive(Default)] +struct Probe { + delayed_ready: AtomicBool, + started_before_ready: AtomicBool, + finished_before_ready: AtomicBool, + heavy_executions: AtomicUsize, +} + +struct SetupDelay { + delayed_task: Option, + probe: Arc, +} + +#[async_trait] +impl RouteTaskHandler for SetupDelay { + async fn handle(&self, event: RouteTaskEvent<'_>) -> Option> { + let delayed = self.delayed_task == Some(event.task_key.task_number); + if delayed { + tokio::time::sleep(SETUP_DELAY).await; + } + let response = UrlEmitterRouteTaskHandler.handle(event).await?; + if delayed || self.delayed_task.is_none() { + self.probe.delayed_ready.store(true, Ordering::SeqCst); + } + Some(response) + } +} + +fn cpu_work(probe: Option>) -> ScalarUDF { + create_udf( + "cpu_work", + vec![DataType::Int64], + DataType::Int64, + Volatility::Volatile, + Arc::new(move |args| { + let Some(probe) = &probe else { + return exec_err!("cpu_work must execute on a remote worker"); + }; + let input = args[0].clone().into_array(1)?; + let input = input.as_any().downcast_ref::().unwrap(); + for task in input.values() { + if *task == 0 { + probe.heavy_executions.fetch_add(1, Ordering::SeqCst); + probe.started_before_ready.store( + !probe.delayed_ready.load(Ordering::SeqCst), + Ordering::SeqCst, + ); + let mut value = 1u64; + for _ in 0..100_000_000 { + value = black_box(value.wrapping_mul(17).wrapping_add(1)); + } + probe.finished_before_ready.store( + !probe.delayed_ready.load(Ordering::SeqCst), + Ordering::SeqCst, + ); + } + } + Ok(args[0].clone()) + }), + ) +} + +criterion_group!(benches, startup_overlap); +criterion_main!(benches); From de765e853567168dc39a9de983af0ad5a46725cc Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Wed, 11 Nov 2026 23:01:51 -0800 Subject: [PATCH 05/31] quote CI run commands ending with trailing :: (#878) #868 added unquoted `run:` commands ending with `::` in `.github/workflows/ci.yml`. Because YAML treats a trailing `:` before a newline as a mapping value indicator, `.github/workflows/ci.yml` has been failing workflow parsing before any jobs start. Quote those `run:` commands so `.github/workflows/ci.yml` parses cleanly. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2cab4487..2d5a9cb2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_correctness_test:: + - run: "cargo test -p datafusion-distributed --features tpch --test integration tpch_correctness_test::" env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -62,7 +62,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_correctness_test:: + - run: "cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_correctness_test::" env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -71,14 +71,14 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_plans_test:: + - run: "cargo test -p datafusion-distributed --features tpch --test integration tpch_plans_test::" tpch-sorted-plans-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_plans_test:: + - run: "cargo test -p datafusion-distributed --features tpch --test integration tpch_sorted_plans_test::" tpcds-correctness-test: runs-on: ubuntu-latest @@ -99,7 +99,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpcds --test integration tpcds_plans_test:: + - run: "cargo test -p datafusion-distributed --features tpcds --test integration tpcds_plans_test::" clickbench-correctness-test: runs-on: ubuntu-latest @@ -114,7 +114,7 @@ jobs: with: path: testdata/clickbench_parquet/ key: "clickbench_parquet" - - run: cargo test -p datafusion-distributed --features clickbench --test integration clickbench_correctness_test:: + - run: "cargo test -p datafusion-distributed --features clickbench --test integration clickbench_correctness_test::" env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -127,7 +127,7 @@ jobs: with: path: testdata/clickbench_parquet/ key: "clickbench_parquet" - - run: cargo test -p datafusion-distributed --features clickbench --test integration clickbench_plans_test:: + - run: "cargo test -p datafusion-distributed --features clickbench --test integration clickbench_plans_test::" iceberg-test: runs-on: ubuntu-latest @@ -145,7 +145,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test -p datafusion-distributed --features tpch,chaos-tests --test integration chaos:: + - run: "cargo test -p datafusion-distributed --features tpch,chaos-tests --test integration chaos::" env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} From a976e71dbc25c1278ea7553917b409e03bf001cf Mon Sep 17 00:00:00 2001 From: alexanderbianchi <75697973+alexanderbianchi@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:11:54 -0500 Subject: [PATCH 06/31] fix: preserve constant metadata across network boundaries (#876) Distributed window queries can fail with `Expects PARTITION BY expression to be ordered` when a partition key is a projected literal. Planning correctly omits the constant key from the sort, but `DistributedCodec` drops its constant metadata when reconstructing the worker plan. Preserve `AcrossPartitions` metadata in the existing equivalence-class protobuf message for shuffle, coalesce, and broadcast boundaries. The additive field distinguishes per-partition constants, uniform constants with unknown values, and known values (including null). Reproducer using the repository's `weather` fixture and two localhost workers: ```sql SELECT typ, "RainToday", "WindGustDir" FROM ( SELECT typ, "RainToday", "WindGustDir", row_number() OVER (PARTITION BY typ, "RainToday", "WindGustDir") AS rn FROM (SELECT 'weather' AS typ, * FROM weather) WHERE "RainToday" = 'No' AND "WindGustDir" IN ('N', 'S') ) WHERE rn = 1 ORDER BY "WindGustDir"; ``` The integration regression fails with the original error on `main` and returns the two expected rows with this patch. The codec round-trip test covers all constant variants across all three boundary types. Validation: - `cargo test -p datafusion-distributed --features integration` - `cargo clippy -p datafusion-distributed --all-targets --all-features -- -D warnings` - `cargo fmt --all -- --check` --- .../src/codec/distributed_codec.rs | 104 +++++++++++++++--- datafusion/distributed/tests/main.rs | 1 + datafusion/distributed/tests/window.rs | 45 ++++++++ 3 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 datafusion/distributed/tests/window.rs diff --git a/datafusion/distributed/src/codec/distributed_codec.rs b/datafusion/distributed/src/codec/distributed_codec.rs index 85cd9b623..2e53861d9 100644 --- a/datafusion/distributed/src/codec/distributed_codec.rs +++ b/datafusion/distributed/src/codec/distributed_codec.rs @@ -15,7 +15,9 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, internal_datafusion_err}; use datafusion::error::DataFusionError; use datafusion::execution::TaskContext; -use datafusion::physical_expr::equivalence::{EquivalenceClass, EquivalenceGroup}; +use datafusion::physical_expr::equivalence::{ + AcrossPartitions, ConstExpr, EquivalenceClass, EquivalenceGroup, +}; use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::union::UnionExec; @@ -513,11 +515,29 @@ fn serialize_equivalence_group( .eq_group() .iter() .map(|class| { - class + let expressions = class .iter() .map(|expr| proto_converter.physical_expr_to_proto(expr, codec)) - .collect::>>() - .map(|expressions| EquivalenceClassProto { expressions }) + .collect::>>()?; + let constant = match class + .canonical_expr() + .and_then(|expr| properties.eq_group().is_expr_constant(expr)) + { + None => None, + Some(AcrossPartitions::Heterogeneous) => { + Some(AcrossPartitionsProto::Heterogeneous(())) + } + Some(AcrossPartitions::Uniform(None)) => { + Some(AcrossPartitionsProto::UniformUnknown(())) + } + Some(AcrossPartitions::Uniform(Some(value))) => { + Some(AcrossPartitionsProto::UniformValue((&value).try_into()?)) + } + }; + Ok(EquivalenceClassProto { + expressions, + constant, + }) }) .collect() } @@ -528,22 +548,34 @@ fn parse_equivalence_properties( decode_ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let classes = equivalence_classes - .into_iter() - .map(|class| { - class - .expressions - .iter() - .map(|expr| { - proto_converter.proto_to_physical_expr(expr, schema.as_ref(), decode_ctx) - }) - .collect::>>() - .map(EquivalenceClass::new) - }) - .collect::>>()?; + let mut classes = Vec::with_capacity(equivalence_classes.len()); + let mut constants = Vec::new(); + for class in equivalence_classes { + let expressions = class + .expressions + .iter() + .map(|expr| proto_converter.proto_to_physical_expr(expr, schema.as_ref(), decode_ctx)) + .collect::>>()?; + if let Some(constant) = class.constant { + let expr = expressions.first().ok_or_else(|| { + internal_datafusion_err!("Constant equivalence class has no expressions") + })?; + let across = match constant { + AcrossPartitionsProto::Heterogeneous(()) => AcrossPartitions::Heterogeneous, + AcrossPartitionsProto::UniformUnknown(()) => AcrossPartitions::Uniform(None), + AcrossPartitionsProto::UniformValue(value) => { + AcrossPartitions::Uniform(Some((&value).try_into()?)) + } + }; + constants.push(ConstExpr::new(Arc::clone(expr), across)); + } + classes.push(EquivalenceClass::new(expressions)); + } let mut properties = EquivalenceProperties::new(schema); properties.add_equivalence_group(EquivalenceGroup::new(classes))?; + // Singleton column classes are otherwise discarded as trivial during reconstruction. + properties.add_constants(constants)?; Ok(properties) } @@ -663,6 +695,19 @@ pub struct EquivalenceClassProto { /// Expressions known to produce equal values. #[prost(message, repeated, tag = "1")] expressions: Vec, + /// Constant metadata can exist without a literal in the expression list. + #[prost(oneof = "AcrossPartitionsProto", tags = "2, 3, 4")] + constant: Option, +} + +#[derive(Clone, PartialEq, ::prost::Oneof)] +pub enum AcrossPartitionsProto { + #[prost(message, tag = "2")] + Heterogeneous(()), + #[prost(message, tag = "3")] + UniformUnknown(()), + #[prost(message, tag = "4")] + UniformValue(protobuf::ScalarValue), } #[derive(Clone, PartialEq, ::prost::Message)] @@ -810,6 +855,7 @@ mod tests { const DEFAULT_MODE: ShuffleMode = ShuffleMode::Direct; use datafusion::arrow::datatypes::{DataType, Field}; + use datafusion::common::ScalarValue; use datafusion::physical_expr::{LexOrdering, PhysicalExpr}; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::filter::FilterExec; @@ -1198,6 +1244,10 @@ mod tests { let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Int32, false), Field::new("b", DataType::Int32, false), + Field::new("c0", DataType::Int32, true), + Field::new("c1", DataType::Int32, true), + Field::new("c2", DataType::Int32, true), + Field::new("c3", DataType::Int32, true), ])); let a: Arc = Arc::new(Column::new("a", 0)); let b: Arc = Arc::new(Column::new("b", 1)); @@ -1206,6 +1256,21 @@ mod tests { equivalence_properties.add_equal_conditions(a.clone(), b.clone())?; equivalence_properties.add_ordering([PhysicalSortExpr::new_default(a.clone())]); + let constants = [ + AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(7)))), + AcrossPartitions::Uniform(Some(ScalarValue::Int32(None))), + AcrossPartitions::Uniform(None), + AcrossPartitions::Heterogeneous, + ]; + let constants = constants + .into_iter() + .enumerate() + .map(|(i, across)| { + ConstExpr::new(Arc::new(Column::new(&format!("c{i}"), i + 2)), across) + }) + .collect::>(); + equivalence_properties.add_constants(constants.clone())?; + let plans: Vec<(&str, Arc)> = vec![ ( "shuffle", @@ -1251,6 +1316,11 @@ mod tests { .exprs_equal(&a, &b), "{name} lost the equivalence relationship" ); + assert_eq!( + decoded.properties().equivalence_properties().constants(), + constants, + "{name} lost constant metadata" + ); if name == "shuffle" { assert_eq!( decoded.properties().output_ordering(), diff --git a/datafusion/distributed/tests/main.rs b/datafusion/distributed/tests/main.rs index bfe99619a..d3e5e896b 100644 --- a/datafusion/distributed/tests/main.rs +++ b/datafusion/distributed/tests/main.rs @@ -32,5 +32,6 @@ mod tpch_plans_test; mod tpch_sorted_correctness_test; mod tpch_sorted_plans_test; mod udfs; +mod window; mod work_unit_feed; mod worker_plan_hook; diff --git a/datafusion/distributed/tests/window.rs b/datafusion/distributed/tests/window.rs new file mode 100644 index 000000000..79230b98e --- /dev/null +++ b/datafusion/distributed/tests/window.rs @@ -0,0 +1,45 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::common::Result; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot}; + + #[tokio::test] + async fn window_partition_by_projected_constant() -> Result<()> { + let (ctx, _guard, _) = start_localhost_context(2, DefaultSessionBuilder).await; + register_parquet_tables(&ctx).await?; + + // Only WindGustDir needs sorting: typ is projected from a literal and RainToday + // is fixed by the filter. Workers must retain both constant facts. + let result = ctx + .sql( + r#" + SELECT typ, "RainToday", "WindGustDir" + FROM ( + SELECT typ, "RainToday", "WindGustDir", + row_number() OVER (PARTITION BY typ, "RainToday", "WindGustDir") AS rn + FROM (SELECT 'weather' AS typ, * FROM weather) + WHERE "RainToday" = 'No' AND "WindGustDir" IN ('N', 'S') + ) + WHERE rn = 1 + ORDER BY "WindGustDir" + "#, + ) + .await? + .collect() + .await?; + + let result = pretty_format_batches(&result)?.to_string(); + assert_snapshot!(result, @r" + +---------+-----------+-------------+ + | typ | RainToday | WindGustDir | + +---------+-----------+-------------+ + | weather | No | N | + | weather | No | S | + +---------+-----------+-------------+ + "); + Ok(()) + } +} From 7fed65b5fb1ea4c50460412ab31ea856df83e87a Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:49:18 +0100 Subject: [PATCH 07/31] Fix insta workspace (#885) Just a small fix to the insta settings --- datafusion/distributed/src/test_utils/insta.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/distributed/src/test_utils/insta.rs b/datafusion/distributed/src/test_utils/insta.rs index cbfebe7b2..ac8863b3c 100644 --- a/datafusion/distributed/src/test_utils/insta.rs +++ b/datafusion/distributed/src/test_utils/insta.rs @@ -12,10 +12,11 @@ macro_rules! assert_snapshot { } pub fn settings() -> insta::Settings { + let root = super::workspace_root().canonicalize().unwrap(); + // `file!()` paths are relative to the workspace root, not to this crate's manifest dir. // Safety: this is only used in tests, it may panic if used in parallel with other tests. - unsafe { env::set_var("INSTA_WORKSPACE_ROOT", env!("CARGO_MANIFEST_DIR")) }; + unsafe { env::set_var("INSTA_WORKSPACE_ROOT", &root) }; let mut settings = insta::Settings::clone_current(); - let root = super::workspace_root().canonicalize().unwrap(); let root = root.to_str().unwrap(); settings.add_filter(root.trim_start_matches("/"), ""); settings.add_filter( From c43b42d013b212e533b5c9f2bf3fda4978cadc95 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:07:30 +0100 Subject: [PATCH 08/31] feat: configure dfbench through a YAML file and -c flags (#890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every time a new config option lands in DataFusion or in `DistributedConfig`, benchmarking it means first adding a dedicated flag to `RunOpt` and wiring it into the session builder. That's easy to forget, it keeps `RunOpt` growing, and any option without a flag simply can't be benchmarked. This makes `dfbench run` take config options generically instead. They're plain `key=value` pairs, and any `datafusion.*` or `distributed.*` key works, in either form: ```yaml # .df-bench.yaml (local, gitignored) distributed.broadcast_joins: true datafusion.execution.target_partitions: 10 ``` ```shell dfbench run --dataset $DATASET -c distributed.broadcast_joins=true -c datafusion.execution.target_partitions=10 ``` The file defaults to `.df-bench.yaml` in the current directory, and `--config-path` points to a different one. Each entry goes through DataFusion's own config setter, in this order, so later sources win: ``` DATAFUSION_* env vars ──▶ .df-bench.yaml (or --config-path) ──▶ -c key=value ... (lowest) (file order) (highest) │ ▼ ConfigOptions::set(key, value) unknown keys fail the run ``` Options that aren't DataFusion configs, such as `--workers`, `-i`, `--debug` and `--work-unit-file-scan`, stay as flags. All the flags that only set a config option are gone, for example `--partitions`, `--batch-size`, `--compression` and `--dynamic`. Their replacement is the matching config key. Metrics collection is on by default, since `--debug` and dynamic task count read the collected metrics. Use `distributed.collect_metrics: false` to turn it off. --- .agents/skills/benchmark/SKILL.md | 14 +- .gitignore | 1 + Cargo.lock | 20 +++ datafusion/distributed-benchmarks/Cargo.toml | 1 + datafusion/distributed-benchmarks/README.md | 29 +++- datafusion/distributed-benchmarks/src/run.rs | 153 +++++++++--------- .../distributed-iceberg-benchmarks/README.md | 2 +- 7 files changed, 133 insertions(+), 87 deletions(-) diff --git a/.agents/skills/benchmark/SKILL.md b/.agents/skills/benchmark/SKILL.md index c2300d62c..17554ff83 100644 --- a/.agents/skills/benchmark/SKILL.md +++ b/.agents/skills/benchmark/SKILL.md @@ -16,7 +16,9 @@ DATASET=$(target/release/dfbench prepare-tpch --scale-factor 10) # only the pa target/release/dfbench run --dataset $DATASET --workers 8 -i 5 ``` -`dfbench run --help` lists all options. +`dfbench run --help` lists all options. Config options (`datafusion.*`, `distributed.*`) come from +a local, gitignored `.df-bench.yaml` in the current directory (or `--config-path `), +overridden by repeated `-c key=value` flags, e.g. `-c distributed.dynamic_task_count=true`. ## Iceberg @@ -48,8 +50,16 @@ statistics. See `datafusion/distributed-benchmarks/README.md` for S3 output and so a profiler attached to `dfbench` sees everything. - Each run stores results under `testdata//.results//` and compares against the previous run. Running on `main` overwrites `main`'s baseline. +- `dfbench run` already repeats every query `-i` times and reports the p50, so one run per + configuration is enough; never repeat whole runs in rounds. For more reliable numbers, raise + `-i` (e.g. `-i 10`). - Background load (other builds or tests, indexing) skews results heavily. Check it before - trusting numbers, and alternate the sides of a comparison over several rounds. + trusting numbers. +- Criterion benches (`cargo bench -p datafusion-distributed-benchmarks --bench `) already + take many samples per case, so run each side once instead of repeating runs: filter to the + case you care about (e.g. `-- 'plan_dump/generate/0$'`), save one side with + `-- --save-baseline ` and compare the other with `-- --baseline `. Benches build + their fixtures for every case even when filtered, so a full run is much slower than one case. - CPU profiles: record with [samply](https://github.com/mstange/samply) and query them from the terminal with Mozilla's [profiler-cli](https://github.com/firefox-devtools/profiler/tree/main/profiler-cli) (`npm install -g @firefox-devtools/profiler-cli@latest`; run `profiler-cli guide` first): diff --git a/.gitignore b/.gitignore index 9c9baee8c..21b6e9039 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ testdata/clickbench/* !testdata/clickbench/queries codegen/target/* codegen/Cargo.lock +.df-bench* diff --git a/Cargo.lock b/Cargo.lock index d8fcf1fcc..9d367d40d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2456,6 +2456,7 @@ dependencies = [ "parquet", "serde", "serde_json", + "serde_yaml_ng", "sketches-ddsketch", "structopt", "tokio", @@ -7046,6 +7047,19 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap 2.14.2", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.7" @@ -8104,6 +8118,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index 5245bb1ff..ccbb112e2 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -17,6 +17,7 @@ parquet = { version = "59" } structopt = { version = "0.3.26" } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.141" +serde_yaml_ng = "0.10" env_logger = "0.11.8" futures = "0.3.31" arrow = { version = "59", features = ["test_utils"] } diff --git a/datafusion/distributed-benchmarks/README.md b/datafusion/distributed-benchmarks/README.md index 21aa8cd81..f44e82c8f 100644 --- a/datafusion/distributed-benchmarks/README.md +++ b/datafusion/distributed-benchmarks/README.md @@ -111,7 +111,8 @@ cargo run -p datafusion-distributed-benchmarks --release -- run --dataset $DATAS The same command is used for running distributed benchmarks: ```shell -cargo run -p datafusion-distributed-benchmarks --release -- run --workers 8 --dataset $DATASET --file-scan-config-bytes-per-partition 16777216 +cargo run -p datafusion-distributed-benchmarks --release -- run --workers 8 --dataset $DATASET \ + -c distributed.file_scan_config_bytes_per_partition=16777216 ``` - `--workers`: Amount of localhost workers used in the query. They all run in @@ -120,9 +121,25 @@ cargo run -p datafusion-distributed-benchmarks --release -- run --workers 8 --da attached to the benchmark process observes both the coordinator and the workers on the same threads. - `--dataset`: Dataset path, as above. -- `--file-scan-config-bytes-per-partition`: How many bytes each partition is - expected to scan. Lower values produce more partitions/tasks. Defaults to the - engine default when unset. +- `-c key=value`: Sets any `datafusion.*` or `distributed.*` config option, and + can be repeated. These take precedence over the config file. + +### Config options + +Config options are read from the `.df-bench.yaml` file in the current directory +when present, or from the file given with `--config-path`. It holds flat +`key: value` pairs, applied in file order on top of any `DATAFUSION_*` +environment variables, and `-c key=value` flags are applied last: + +```yaml +datafusion.execution.target_partitions: 8 +distributed.broadcast_joins: true +``` + +`distributed.dynamic_plan_remainder_replanning` only takes effect together with +`distributed.dynamic_task_count: true`. Metrics are collected by default, as +`--debug` and dynamic task count read them; set `distributed.collect_metrics: false` +to turn them off. ### Iceberg benchmarks @@ -164,9 +181,9 @@ Run each representation with its format-specific binary, then compare them with ```shell cargo run -p datafusion-distributed-benchmarks --release -- run \ - --workers 2 --dataset $DATASET --partitions 2 + --workers 2 --dataset $DATASET -c datafusion.execution.target_partitions=2 cargo run -p datafusion-distributed-iceberg-benchmarks --release -- run \ - --workers 2 --dataset $ICEBERG --partitions 2 + --workers 2 --dataset $ICEBERG -c datafusion.execution.target_partitions=2 dfbench compare $DATASET $ICEBERG dfbench compare $DATASET@base $ICEBERG@candidate diff --git a/datafusion/distributed-benchmarks/src/run.rs b/datafusion/distributed-benchmarks/src/run.rs index 7bc5f1084..905343b0d 100644 --- a/datafusion/distributed-benchmarks/src/run.rs +++ b/datafusion/distributed-benchmarks/src/run.rs @@ -19,12 +19,10 @@ use crate::backend::BenchmarkBackend; use crate::local_workers::LocalWorkers; use crate::results::{BenchResult, BenchmarkRun, QueryIter, dataset_path}; use crate::stats::stats_estimation_q_error; -use datafusion::arrow::ipc::CompressionType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::instant::Instant; use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::common::utils::get_available_parallelism; -use datafusion::common::{config_err, exec_err, not_impl_err}; +use datafusion::common::{config_datafusion_err, config_err, exec_err, not_impl_err}; use datafusion::datasource::source::DataSourceExec; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::SessionStateBuilder; @@ -37,9 +35,10 @@ use datafusion_distributed::test_utils::work_unit_file_scan::{ work_unit_file_scan_desired_task_count, work_unit_file_scan_scale_up_leaf_node, }; use datafusion_distributed::{ - DistributedExt, DistributedMetricsFormat, NetworkBoundaryExt, SessionStateBuilderExt, - display_plan_ascii, rewrite_distributed_plan_with_metrics, + DistributedConfig, DistributedExt, DistributedMetricsFormat, NetworkBoundaryExt, + SessionStateBuilderExt, display_plan_ascii, rewrite_distributed_plan_with_metrics, }; +use serde_yaml_ng::{Mapping, Value}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -72,43 +71,10 @@ pub struct RunOpt { #[structopt(long, default_value = "0")] workers: usize, - /// Number of bytes each partition is expected to scan from parquet files. Lower values - /// produce more partitions/tasks. Defaults to the engine default when unset. - #[structopt(long)] - file_scan_config_bytes_per_partition: Option, - - /// Task count scale factor for when nodes in stages change the cardinality of the data - #[structopt(long)] - cardinality_task_sf: Option, - - /// Collects metrics across network boundaries - #[structopt(long)] - collect_metrics: bool, - - /// Collects metrics across network boundaries - #[structopt(long, default_value = "lz4")] - compression: String, - - /// Sets the limits of tasks for each stage - #[structopt(long, default_value = "0")] - max_tasks_per_stage: usize, - - /// Activate dynamic task count - #[structopt(long)] - dynamic: bool, - /// Number of iterations of each test run #[structopt(short = "i", long = "iterations", default_value = "5")] iterations: usize, - /// Number of partitions to process in parallel. Defaults to the number of available cores. - #[structopt(short = "n", long = "partitions")] - partitions: Option, - - /// Batch size when reading CSV or Parquet files - #[structopt(short = "s", long = "batch-size")] - batch_size: Option, - /// Activate debug mode to see more details #[structopt(short, long)] debug: bool, @@ -119,13 +85,17 @@ pub struct RunOpt { #[structopt(long = "work-unit-file-scan")] work_unit_file_scan: bool, - /// Fan-out threshold for choosing between Direct and TwoPhase shuffle modes. In Direct mode - /// the producer hashes into `consumer_task_count × consumer_partition_count` global buckets; - /// TwoPhase mode is used when that product meets or exceeds this value, reducing the - /// producer-side hash to `consumer_task_count` buckets and adding a local RepartitionExec - /// on the consumer side. - #[structopt(long = "two-step-shuffle-fanout-threshold", default_value = "128")] - two_step_shuffle_fanout_threshold: usize, + /// YAML file with DataFusion and distributed config options as flat `key: value` pairs, such + /// as `datafusion.execution.target_partitions: 8` or `distributed.broadcast_joins: true`. + /// Defaults to `.df-bench.yaml` in the current directory, which is skipped if it doesn't exist. + #[structopt(long)] + config_path: Option, + + /// A DataFusion or distributed config option as `key=value`, such as + /// `-c distributed.dynamic_task_count=true`. Can be repeated, and takes precedence over the + /// values in the config file. + #[structopt(short = "c", long = "config", parse(try_from_str = parse_config_entry))] + config: Vec<(String, String)>, } /// The query suite of a dataset path such as `tpch_parquet/scale_factor=1/...`: the benchmark @@ -161,14 +131,54 @@ fn queries_for_dataset(dataset: &str) -> Result, DataFusio } } +/// The default `--config-path`, relative to the current directory. +const DEFAULT_CONFIG_PATH: &str = ".df-bench.yaml"; + +fn parse_config_entry(entry: &str) -> Result<(String, String), String> { + let (key, value) = entry + .split_once('=') + .ok_or_else(|| format!("expected `key=value`, got `{entry}`"))?; + Ok((key.trim().to_string(), value.trim().to_string())) +} + +/// Reads the flat `key: value` config entries of a YAML file, in file order. +fn read_config_file(path: &Path) -> Result> { + let yaml = fs::read_to_string(path) + .map_err(|e| config_datafusion_err!("Cannot read {}: {e}", path.display()))?; + // A file with only comments is a null document. + let entries: Option = serde_yaml_ng::from_str(&yaml) + .map_err(|e| config_datafusion_err!("Invalid YAML in {}: {e}", path.display()))?; + entries + .unwrap_or_default() + .into_iter() + .map(|(key, value)| { + let Value::String(key) = key else { + return config_err!("Config keys in {} must be strings", path.display()); + }; + let value = match value { + Value::Bool(v) => v.to_string(), + Value::Number(v) => v.to_string(), + Value::String(v) => v, + v => return config_err!("Config value of {key} must be a scalar, got {v:?}"), + }; + Ok((key, value)) + }) + .collect() +} + impl RunOpt { - fn config(&self) -> Result { - SessionConfig::from_env().map(|mut config| { - if let Some(batch_size) = self.batch_size { - config = config.with_batch_size(batch_size); + /// The config entries of the config file followed by the `--config` ones, so that applying + /// them in order makes the `--config` ones take precedence. + fn config_entries(&self) -> Result> { + let mut entries = match &self.config_path { + Some(path) => read_config_file(path)?, + None if Path::new(DEFAULT_CONFIG_PATH).exists() => { + read_config_file(Path::new(DEFAULT_CONFIG_PATH))? } - config.with_target_partitions(self.partitions.unwrap_or_else(get_available_parallelism)) - }) + None => vec![], + }; + entries.extend(self.config.iter().cloned()); + Ok(entries) } pub async fn run(self, mut backend: B) -> Result<()> { @@ -185,30 +195,11 @@ impl RunOpt { ) -> Result<()> { let mut builder = SessionStateBuilder::new() .with_default_features() - .with_config(self.config()?) + .with_config(SessionConfig::from_env()?) .with_distributed_worker_resolver(LocalHostWorkerResolver::new( local_workers.ports.to_vec(), )) .with_distributed_planner() - .with_distributed_cardinality_effect_task_scale_factor( - self.cardinality_task_sf.unwrap_or(1.0), - )? - .with_distributed_dynamic_task_count(self.dynamic)? - .with_distributed_compression(match self.compression.as_str() { - "zstd" => Some(CompressionType::ZSTD), - "lz4" => Some(CompressionType::LZ4_FRAME), - "none" => None, - v => return config_err!("Unknown compression type {v}"), - })? - .with_distributed_children_isolator_unions(true)? - .with_distributed_broadcast_joins(true)? - .with_distributed_two_step_shuffle_fanout_threshold( - self.two_step_shuffle_fanout_threshold, - )? - .with_distributed_metrics_collection( - self.collect_metrics || self.debug || self.dynamic, - )? - .with_distributed_max_tasks_per_stage(self.max_tasks_per_stage)? .with_distributed_user_codec(WorkUnitFileScanCodec) .with_distributed_desired_task_count_handler(work_unit_file_scan_desired_task_count) .with_distributed_scale_up_leaf_node_handler(work_unit_file_scan_scale_up_leaf_node) @@ -216,17 +207,20 @@ impl RunOpt { dse.data_source() .downcast_ref::() .map(|v| &v.feed) - }); + }) + .with_distributed_metrics_collection(true)?; + + let config_entries = self.config_entries()?; + let options = builder.config().get_or_insert_default().options_mut(); + for (key, value) in &config_entries { + options.set(key, value)?; + } if let Some(local_worker) = local_workers.workers.first() { let url = Url::parse(&format!("http://127.0.0.1:{}", local_workers.ports[0])).unwrap(); builder.set_distributed_local_worker_context(local_worker.to_local_worker_context(url)) } - if let Some(bytes_per_partition) = self.file_scan_config_bytes_per_partition { - builder.set_distributed_file_scan_config_bytes_per_partition(bytes_per_partition)?; - } - if self.work_unit_file_scan { builder = builder.with_physical_optimizer_rule(Arc::new(WorkUnitFileScanRule)) } @@ -237,6 +231,7 @@ impl RunOpt { let dataset_suite = dataset_suite(&self.dataset); println!("Running benchmarks with the following options: {self:?}"); + println!("Config: {config_entries:?}"); let mut benchmark_run = BenchmarkRun::new(self.dataset.clone(), local_workers.ports.len()); for (id, sql) in queries_for_dataset(dataset_suite)? { @@ -262,6 +257,8 @@ impl RunOpt { sql: &str, ctx: &SessionContext, ) -> Result { + let dynamic = + DistributedConfig::from_session_config(&ctx.copied_config())?.dynamic_task_count; let mut bench_query = BenchResult { id: id.to_string(), dataset: self.dataset.clone(), @@ -284,7 +281,7 @@ impl RunOpt { let elapsed = start.elapsed(); let ms = elapsed.as_secs_f64() * 1000.0; let row_count = result.iter().map(|b| b.num_rows()).sum(); - let physical_plan = if self.dynamic || self.debug { + let physical_plan = if self.debug || dynamic { rewrite_distributed_plan_with_metrics( physical_plan, DistributedMetricsFormat::PerTask, @@ -293,7 +290,7 @@ impl RunOpt { } else { physical_plan }; - let stats_q_error = match self.dynamic { + let stats_q_error = match dynamic { true => stats_estimation_q_error(&physical_plan), false => None, }; diff --git a/datafusion/distributed-iceberg-benchmarks/README.md b/datafusion/distributed-iceberg-benchmarks/README.md index 963056d27..66ab27d62 100644 --- a/datafusion/distributed-iceberg-benchmarks/README.md +++ b/datafusion/distributed-iceberg-benchmarks/README.md @@ -11,7 +11,7 @@ DATASET=$(cargo run -p datafusion-distributed-benchmarks --release -- prepare-tp ICEBERG=$(cargo run -p datafusion-distributed-iceberg-benchmarks --release -- prepare \ --input $DATASET) cargo run -p datafusion-distributed-iceberg-benchmarks --release -- run \ - --workers 2 --dataset $ICEBERG --partitions 2 + --workers 2 --dataset $ICEBERG -c datafusion.execution.target_partitions=2 ``` Preparation writes an immutable `tpch_iceberg` dataset by default, whose path repeats the input's From 9e37cab0be7a8be30a2d445b54ba951d22d3a0dc Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:21:29 +0100 Subject: [PATCH 09/31] feat: add plan dumps that can be logged and displayed with `dfd plan` (#841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > [!WARNING] > This PR is big, and it's exclusively dev tooling with no implications in the production code. A very light review is recommended, as the idea is that LLMs are the main ones maintaining this tool. Closes https://github.com/datafusion-contrib/datafusion-distributed/issues/523 Ships a tool + a SKILL.md for LLMs (and humans) to debug execution plans and identify bottlenecks and weak spots. There's two main pieces this PR ships: - A plan dump generator function, `generate_plan_dump(plan)`, that generates a compressed logeable b64 string for people to add to their own observability layer. - A CLI for inspecting that plan (+ a SKILL.md to use it) ## Dump size Dumps are meant to live in logs, so they need to be small in size and CPU footprint. Here's a table that compares logging the ASCII plan (even if it was zstd-compressed and base64-encoded) with the plan dumps proposed in this PR: | Query | ASCII plan: bytes (CPU time) | ASCII + zstd + base64: bytes (CPU time) | Plan dump (proto + zstd + base64): bytes (CPU time) | |---|---:|---:|---:| | q1 | 40,614 (846 µs) | 6,716 (949 µs) | 3,640 (485 µs) | | q2 | 227,396 (3,544 µs) | 26,040 (4,184 µs) | 16,112 (1,701 µs) | | q3 | 101,454 (1,581 µs) | 13,436 (1,853 µs) | 6,948 (890 µs) | | q4 | 74,524 (1,300 µs) | 12,236 (1,590 µs) | 6,884 (712 µs) | | q5 | 158,965 (2,515 µs) | 18,984 (2,976 µs) | 10,036 (1,202 µs) | | q6 | 33,105 (573 µs) | 4,500 (656 µs) | 1,592 (419 µs) | | q7 | 175,123 (2,878 µs) | 22,648 (3,539 µs) | 13,024 (1,463 µs) | | q8 | 211,001 (3,358 µs) | 23,412 (3,843 µs) | 12,744 (1,533 µs) | | q9 | 175,677 (2,906 µs) | 23,828 (4,039 µs) | 14,008 (1,688 µs) | | q10 | 121,009 (1,948 µs) | 16,032 (2,254 µs) | 8,528 (974 µs) | | q11 | 154,003 (2,491 µs) | 18,296 (2,795 µs) | 10,524 (1,106 µs) | | q12 | 72,108 (1,150 µs) | 9,644 (1,344 µs) | 4,968 (665 µs) | | q13 | 79,315 (1,473 µs) | 13,884 (1,737 µs) | 8,096 (716 µs) | | q14 | 59,553 (844 µs) | 7,856 (992 µs) | 3,452 (507 µs) | | q16 | 105,161 (1,892 µs) | 15,836 (2,283 µs) | 9,008 (1,040 µs) | | q17 | 97,139 (1,454 µs) | 12,872 (1,717 µs) | 7,176 (762 µs) | | q18 | 129,084 (2,060 µs) | 17,200 (2,315 µs) | 9,468 (954 µs) | | q19 | 93,076 (1,455 µs) | 8,608 (1,577 µs) | 4,404 (1,111 µs) | | q20 | 154,741 (2,621 µs) | 22,196 (3,075 µs) | 13,080 (1,320 µs) | | q21 | 183,115 (3,058 µs) | 23,992 (3,564 µs) | 14,296 (1,548 µs) | | q22 | 94,338 (2,954 µs) | 12,200 (3,344 µs) | 6,576 (2,187 µs) | | **Total** | **2,540,501 (42,901 µs)** | **330,416 (50,626 µs)** | **184,564 (22,983 µs)** | ## Usage ### 1. Log a dump from your service Once the query has been fully executed, fold the worker metrics back into the plan and log its dump next to the rest of the query's telemetry: ```rust let plan = ctx.sql(sql).await?.create_physical_plan().await?; let batches = execute_stream(plan.clone(), ctx.task_ctx())?.try_collect::>().await?; let plan = rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::PerTask).await?; tracing::info!(plan_dump = %generate_plan_dump(&plan)?, "query finished"); ``` A dump is a single base64 line. Here's a complete one for TPC-H q3 (SF1, 4 workers), 2.9 KB, which you can paste into `dfd plan` to try out the commands below: ``` KLUv/WIAgIcqbUQAY33rHsVztSify0X5XBYV/K4/UC9xHCPTIs6RVSPAHH9P2C7ZNpzK6iri0FonOuvDyIyx6jG0s2FyioWt9KxGx2pI27J9sYJP7el9YhoAN6yp9XwD4OMp25IMew7Lr29yEVsJvCo0JfAyABkANywAhB/3D39Z6Aj6Sjc2z5XUEB2m2g7aDtprmJlNp8fV5NHJWKJD331e9hFenqK10j8O4zcAblg0aWEaIUiQ/pS3SlTlu+sDqUeFT1+xrxCEFhYX1+3uweUmvrUCG6l6WEv2XhoEFEZKkxZG5diW+GC/3xvbFjo7IpWst73saNEV6U4Fjy79ZMlztysl19jNHa8PogR7zDp19SN8wkEN5OUAGqZuQavsnDLFZU4AFV4MbKUzPEQqPhutVGQPxbR7nzL0WlgECZ2jqCmwL4i7aoDwbOJhhf7Kn9i8VLwm8m9SmwuxbWkZ+XtR+bt+Qf326l5Zd7iiqkNTiXm4aYmEUWBMFddSNeV/CDRuJhwHuDyYQ3NANY1/lgVEMDxSyppFTuM/PX26B3NolbCe6riDrUYlGJYSLi0hrhmBOggpMSpr10/W/Zla5l155XVr4FldZbiRzbePOwPeH7ZZH59PELbipneIbTU/Q7M9WI3UjT1+NXN3IqEgbMV9pKmxbjVS+0m2fMPWQ9gKggYyMOAEGDFgEMAKIrxgcc3w02NYHLhqTOl+kk17tzR2oLGpq4T1cP23Q0pySiWspwOzctFHktTI9mVvPMl2V0fLSj5w9fQRJCsOWOfqaJkFO1cEmdEux4vswIWrAsID15M+/SQJTG0jvYEl4Cx7WDtZIE7MQJGY6mVGIM3R1j3u1mPr/q5b27pfIA5s6x7fzbbuve9Ad9q6B6HpbOvew5XY1r2IMp2ER0Ia8l7wBMYgeQWxrXbQLbbZUXRhVRIeCV/wBN43libZ3RMkV00qex9w09q7E/8uz1691ahuNyOtloY8GvJikIxBsmri8nVNAFL3QJ/e6qvENqNQnWE220+IGWpV3C0LvKkYxgRLIOHNBGZaHy85lOhIbEFgYVlpgaHCCSykcMH/uTrQo39w9x+dfdEVgiG9RGIJHQuwZmisFFirpaZVjVyqbNS4FrGqEQsWrAPuDgHI74F1AEJLCvk6QZV+MrXTPUGaLN8/lmc76JN86IX+VOVwlYKoE7/+q841QmWnwqJjLtdX8nwgPbacyKnFBYMhvYNGkMEVjV4QYmABhgwwXsigSP9eilT8onT2XHwitrdX2QaB0aDhJXJKSpQMB9EEBRmCEGNYnQ9OoHCxS+BEFmYdJTW4TUa5jukYygXXc0GS52j5j+4EY1tzq2VZQGplv3LbjIh0YCNUW/1qBlUEUAiAD9MHELbdGrgRHWEFH3smd7w7BVnLr1ZPQO8917vzaNT95vDtlazZTg1QTTYNHk1PkzvA1q8+tFu8Bk0wdDHRXC2hZ2W64qxqqBgkUJAdlwVuJJ2NQlRLJCtbXKhnksfSMfuj/SKv9mwGUHZCTxCEE91qWJ1tU+9N1VLCSgTIHhE5mFZ/lCO/yl8nCo3t0Cd3DPK4++gyHVmcQyFVCicDmtc51MZ2p1GwmgMQmEYEDtKm3qvkakGscgPfromaPBtYPwdk9lwW7QESrg8WIOcNsPz/0adeF+yaaaNdewUNhWDMHO59zvkG+zFK3x2BA+W5AGDV1ozn9T7sxXpmN1buzIL0+K2keR4OPu0QGvU4g/GIkNL10Bx5KNE+nZPRLVqCSss7LsqD561xjPF0vXJyLvtk7Dl1DRpHs5nzNA4CDgfc4482Mvw2o9rYjnPmmorQyrPFjvOcgzhYK6D4+uw4ZQfkp3U/ymd6SCNbbAqS52IXXUiKGhjTaWKHEuWfoewJhU88EdbhEVySuC0vADs0BTOyGU+hRsQB2VJeAiSk7Asr7kaBsPyFhP8hUDX6FpdsKC3C8aDAGD43oAjpOvvtYHjH/xwdnX2BHJ/0dB0b0iFiVUe0nkL3yHnIJAZuSjNfcpokRJ0y+B45XExgyiJf2c0wc9KoPPlADo2Ff2aAj9XPVXRsI8bVj2nglO04bECGGOnhuCzY0h0mUnvDN700tnUmvq1i272wSXNBnucZt9gQAopL7rhm4pINYCLZpdJOgzrXzPhqnPD0gFfcBtnbYRdY0SCJ+3wAGxue3RsTSxd5cXBpAAzOsC7P3NaUIF2DVN5yUyfhXvMO84AqB/7yqGTBa5zKkrmLdQZWY9OJWiablxq9BQorh5Dsbs2EwDlrn2wtZ+73U8M8Y5OcQl0hYU+M8tkbk7S5WZZ1DVQVdITAePji6t0/5SLkuGX8AzzYZv8ppErctZ1JJqKaeOJbroe46DFQVm6F7iVjYBMlURSFoiSKJJHc0ANT+hwuKCvlnoxoBMBYiNEJJ7T0FW7rHPOocX1ctbYtIDMbWpR/snHweZGAEQVKngerTvnXTuA+6rvk+iQZKT31rZsiQAeFywKQEroJENyuyjkJTqIcpHB1QWVBDoSrswSvojkADpK8T1MGv5xUB/JXkp5tJRdxQfNdziwEscQdeZ5nH+7lnlF8Qz275YWpi8ScsJcL8fd4K+ayWmuDsaZ4aBfHHRKd6fB6ORxKSwYu414o49GKQScI/zoqKmwo1BuFhXpsd7IkR3fobCY5qGgrdgsd50/w2Igpj4jV3ZDFPHFiKfmofC4VvAR887cc8RENRbB+JqI1G8KhKFFaBxMjxhrxD2+Rjw9ruGapfTtnqJZWKwjP3ERZvYA7q018i6wTIQTWUYgixmKeDIlZzdbA15F4xq2W7ACSx3EAzxG3PgmSIOdSm2RaLH3jK1vsDGMt3EwIRNDMkuGujdOCke+a5HiGk7FpCA85cpqSWDCEByFdefFTolh97Er1DsoVtULrT1qnhXTkWGQlBLOF2SZrTPPnEQM= ``` You can try this out yourself by checking out the branch of this PR locally and installing the `dfd` CLI with: ``` cargo install --path datafusion/distributed-cli/ pbpaste | dfd plan ``` ### 2. Get an overview of the stages Copy the dump from your logs and pass it to `dfd plan` as an argument, as a file path or through stdin. With no other arguments, it shows the tree of stages with their stage-level metrics, added up across tasks. Some metrics are elided below: ```rust $ pbpaste | dfd plan DistributedExec │ output_rows=11.62 K │ elapsed_compute=5.43ms │ output_bytes=363.1 KB │ … └─ Stage 3 (Coalesce) tasks=2 partitions=16 │ output_rows=11.62 K │ elapsed_compute=77.81ms │ output_bytes=363.1 KB │ … ├─ Stage 1 (Shuffle) tasks=1 partitions=32 │ output_rows=147.1 K │ elapsed_compute=6.61ms │ output_bytes=2.2 MB │ … └─ Stage 2 (Shuffle) tasks=2 partitions=32 output_rows=3.24 M elapsed_compute=21.42ms output_bytes=123.7 MB … ``` ### 3. Inspect a single stage `--stage ` shows the nodes of that stage with their metrics. The network boundaries point at the stages they read from, which you can open next. Stage 0 is the head of the plan, run by the coordinator: ```rust $ pbpaste | dfd plan --stage 3 Stage 3 (Coalesce) tasks=2 partitions=16 │ output_rows=11.62 K │ elapsed_compute=77.81ms │ output_bytes=363.1 KB │ … └─ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] │ output_rows=11.62 K │ elapsed_compute=38.59µs │ output_bytes=363.1 KB │ … └─ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true] │ output_rows=11.62 K │ elapsed_compute=72.29µs │ output_bytes=363.1 KB │ … └─ AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ output_rows=11.62 K │ elapsed_compute=2.97ms │ output_bytes=437.6 KB │ … └─ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] │ output_rows=30.52 K │ elapsed_compute=17.58ms │ output_bytes=12.0 MB │ … ├─ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=1 │ output_rows=147.1 K │ elapsed_compute=1.53ms │ output_bytes=2.2 MB │ … └─ [Stage 2] => NetworkShuffleExec: output_partitions=16, input_tasks=2 output_rows=3.24 M elapsed_compute=55.63ms output_bytes=123.7 MB … ``` ### 4. Filter metrics `--metric ` leaves only the metrics whose name contains that text. It can be repeated, like `-m bytes -m rows`, and `--exact` matches whole names instead. Here's where the CPU time of a stage goes: ```rust $ pbpaste | dfd plan --stage 3 --metric elapsed_compute Stage 3 (Coalesce) tasks=2 partitions=16 │ elapsed_compute=77.81ms └─ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] │ elapsed_compute=38.59µs └─ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true] │ elapsed_compute=72.29µs └─ AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ elapsed_compute=2.97ms └─ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] │ elapsed_compute=17.58ms ├─ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=1 │ elapsed_compute=1.53ms └─ [Stage 2] => NetworkShuffleExec: output_partitions=16, input_tasks=2 elapsed_compute=55.63ms ``` ### 5. Break metrics down per task Metrics are added up across the tasks of a stage. `--verbose` shows the value of each task instead, which tells apart a stage that is slow everywhere from one with a straggler: ```rust $ pbpaste | dfd plan --stage 3 --metric elapsed_compute --verbose Stage 3 (Coalesce) tasks=2 partitions=16 │ elapsed_compute={0:23.59ms, 1:54.22ms} └─ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] │ elapsed_compute={0:26.55µs, 1:12.04µs} └─ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true] │ elapsed_compute={0:35.42µs, 1:36.87µs} └─ AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ elapsed_compute={0:1.92ms, 1:1.04ms} └─ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] │ elapsed_compute={0:8.54ms, 1:9.04ms} ├─ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=1 │ elapsed_compute={1:1.53ms} └─ [Stage 2] => NetworkShuffleExec: output_partitions=16, input_tasks=2 elapsed_compute={0:13.07ms, 1:42.56ms} ``` The output is colored in a terminal. `FORCE_COLOR=1 dfd plan --stage 2 < dump.txt | less -R` keeps the colors when paging through big stages. --- .agents/skills/benchmark/SKILL.md | 6 + .agents/skills/plan-dump/SKILL.md | 50 + AGENTS.md | 1 + Cargo.lock | 3 + datafusion/distributed-benchmarks/Cargo.toml | 4 + .../benches/plan_dump.rs | 83 ++ .../distributed-benchmarks/src/results.rs | 28 +- datafusion/distributed-benchmarks/src/run.rs | 31 +- datafusion/distributed-cli/Cargo.toml | 3 +- datafusion/distributed-cli/README.md | 56 ++ datafusion/distributed-cli/src/main.rs | 4 + datafusion/distributed-cli/src/plan.rs | 74 ++ datafusion/distributed/Cargo.toml | 2 + datafusion/distributed/src/coordinator/mod.rs | 1 + .../src/coordinator/prepare_dynamic_plan.rs | 5 +- datafusion/distributed/src/lib.rs | 7 +- .../src/metrics/generated/metrics.rs | 43 + .../distributed/src/metrics/metrics.proto | 31 + datafusion/distributed/src/metrics/mod.rs | 2 + .../src/metrics/plan_dump/derived_metrics.rs | 180 ++++ .../distributed/src/metrics/plan_dump/mod.rs | 6 + .../src/metrics/plan_dump/plan_dump.zdict | Bin 0 -> 65536 bytes .../metrics/plan_dump/plan_dump_display.rs | 872 ++++++++++++++++++ .../metrics/plan_dump/plan_dump_generation.rs | 360 ++++++++ datafusion/distributed/src/stage.rs | 21 +- .../distributed/src/test_utils/plans.rs | 43 +- 26 files changed, 1875 insertions(+), 41 deletions(-) create mode 100644 .agents/skills/plan-dump/SKILL.md create mode 100644 datafusion/distributed-benchmarks/benches/plan_dump.rs create mode 100644 datafusion/distributed-cli/src/plan.rs create mode 100644 datafusion/distributed/src/metrics/plan_dump/derived_metrics.rs create mode 100644 datafusion/distributed/src/metrics/plan_dump/mod.rs create mode 100644 datafusion/distributed/src/metrics/plan_dump/plan_dump.zdict create mode 100644 datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs create mode 100644 datafusion/distributed/src/metrics/plan_dump/plan_dump_generation.rs diff --git a/.agents/skills/benchmark/SKILL.md b/.agents/skills/benchmark/SKILL.md index 17554ff83..fe8a7e267 100644 --- a/.agents/skills/benchmark/SKILL.md +++ b/.agents/skills/benchmark/SKILL.md @@ -20,6 +20,12 @@ target/release/dfbench run --dataset $DATASET --workers 8 -i 5 a local, gitignored `.df-bench.yaml` in the current directory (or `--config-path `), overridden by repeated `-c key=value` flags, e.g. `-c distributed.dynamic_task_count=true`. +## Plan dumps + +`--debug` prints a plan dump per iteration to stdout (everything else goes to stderr), which +`dfd plan` (`-p datafusion-distributed-cli`) inspects; see the `plan-dump` skill. `dfbench` reads +`DATAFUSION_*` env vars, e.g. `DATAFUSION_OPTIMIZER_JOIN_REORDERING=false`. + ## Iceberg The same benchmarks run over Iceberg tables with the `dfbench-iceberg` binary, which takes the diff --git a/.agents/skills/plan-dump/SKILL.md b/.agents/skills/plan-dump/SKILL.md new file mode 100644 index 000000000..673eb2333 --- /dev/null +++ b/.agents/skills/plan-dump/SKILL.md @@ -0,0 +1,50 @@ +--- +name: plan-dump +description: Inspect executed distributed plans with `dfd plan`. Use when asked why a query is slow, where its time or rows go, how two plans of the same query differ (flags on/off, join reordering, branches), or whether a plan change could be returning wrong results. +--- + +# Inspecting plan dumps with `dfd plan` + +A plan dump is one base64 line holding an executed plan with its stages and per-task metrics, +produced by `generate_plan_dump` after `rewrite_distributed_plan_with_metrics`. `dfd plan` +takes a dump as an argument, a file path, or stdin. + +## Reading a dump + +- **Default view.** With no options, it prints the tree of stages with stage-level metrics. + `--stage N` shows the nodes of one stage. Stage 0 is the head, run by the coordinator. +- **Per-task metrics.** `-v` shows metrics per task rather than aggregated, which is how skew + shows up. +- **Filtering metrics.** `-m TEXT` keeps metrics whose name contains `TEXT`, and is repeatable. + `--exact` matches whole names. + - `-m output_rows --exact` on the stage tree gives a compact skeleton of the whole plan. + - `-m none --exact` hides every metric, leaving plain node lines to grep. +- **Long lines.** Node lines are long, so cut them when scanning. +- **Stage-level metrics:** + - `elapsed_compute` is the CPU time of all the stage's nodes. + - `output_rows` and `output_bytes` are measured where the stage is read: what crossed the + network. + - `plan_added_at`, `plan_executed_at` and `plan_finished_at` are offsets from query start. + They give the timeline and the critical path. + - Dynamic task count adds `estimated_output_bytes` and `output_bytes_q_error`. These show how + wrong the sampled statistics were, and adaptive replanning decides from them. +- **Node metrics worth knowing:** + - On joins: `build_input_rows`, `input_rows` and `probe_hit_rate`. + - On filters: `selectivity`. + +## Comparing two plans of the same query + +- **Stage ids don't necessarily match across runs.** They are renumbered whenever the plan's + shape changes, so match nodes by content: node type plus its keys or expressions (join keys, + grouping, predicates). +- **Broadcast counts multiply.** Broadcast rows are counted once per consumer task, both in a + broadcast stage's `output_rows` and in a `CollectLeft` join's `build_input_rows`. A broadcast + turning into a shuffle drops them by that factor without losing any rows. +- **Logical row counts should match.** A node computing the same thing in both plans (the same + join, final aggregation or filter) should report the same `output_rows`, whatever its physical + shape. When it differs, the results differ, unless a `LIMIT`/TopK above it stops execution + early or dynamic filters prune its input differently. +- **Equal counts don't prove equal results.** Mixed-up columns keep every count intact. Check + the column indices in projections and sort expressions where the plans diverge. +- **Runs can differ.** Adaptive decisions come from sampled runtime statistics, so runs of the + same query can produce different plans. Group runs by plan shape before comparing timings. diff --git a/AGENTS.md b/AGENTS.md index 96365910b..4ff0f8003 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,6 +98,7 @@ Read the relevant contributor guide before starting work in these areas: - Performance and benchmark changes: `docs/source/contributor-guide/04-benchmarks.md` - Running, comparing or profiling benchmarks: `.agents/skills/benchmark/SKILL.md` +- Inspecting executed plans and comparing them (`dfd plan`): `.agents/skills/plan-dump/SKILL.md` - Linking upstream DataFusion/arrow-rs source to debug or fix them: `.agents/skills/upstream-source/SKILL.md` diff --git a/Cargo.lock b/Cargo.lock index 9d367d40d..5d72ad4fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2405,6 +2405,7 @@ dependencies = [ "arrow-ipc", "arrow-select", "async-trait", + "base64 0.22.1", "bytes", "chrono", "crossbeam-queue", @@ -2438,6 +2439,7 @@ dependencies = [ "tower", "url", "uuid", + "zstd 0.13.3", ] [[package]] @@ -2474,6 +2476,7 @@ dependencies = [ "datafusion-distributed", "dirs", "env_logger", + "prost", "tokio", ] diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index ccbb112e2..bdad6aaaf 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -59,6 +59,10 @@ harness = false name = "metrics_collection" harness = false +[[bench]] +name = "plan_dump" +harness = false + [[bench]] name = "startup_overlap" harness = false diff --git a/datafusion/distributed-benchmarks/benches/plan_dump.rs b/datafusion/distributed-benchmarks/benches/plan_dump.rs new file mode 100644 index 000000000..d80a15560 --- /dev/null +++ b/datafusion/distributed-benchmarks/benches/plan_dump.rs @@ -0,0 +1,83 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::collect; +use datafusion_datasets::output::TESTDATA_ROOT; +use datafusion_datasets::parquet_layout::ParquetLayout; +use datafusion_datasets::tpch::TpchSpec; +use datafusion_datasets::{register_tables, tpch}; +use datafusion_distributed::test_utils::localhost::start_localhost_context; +use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, DistributedMetricsFormat, + generate_plan_dump, rewrite_distributed_plan_with_metrics, +}; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::runtime::Builder as RuntimeBuilder; + +const NUM_WORKERS: usize = 10; +const TARGET_PARTITIONS: usize = 128; + +/// Measures [generate_plan_dump] on executed TPC-H plans with their metrics, which is the part of +/// dumping a plan that runs in an application. Executing the queries and collecting their metrics +/// happens once per query, outside of the measurement. +fn plan_dump(c: &mut Criterion) { + let rt = RuntimeBuilder::new_multi_thread() + .enable_all() + .build() + .expect("tokio runtime"); + + let mut group = c.benchmark_group("plan_dump"); + + let tpch_spec = TpchSpec { + location: TESTDATA_ROOT.to_string(), + scale_factor: 0.1, + layout: ParquetLayout::default(), + }; + + let (ctx, _join_set, _workers) = rt + .block_on(async move { + tpch::generate_data(&tpch_spec).await.unwrap(); + + let (ctx, join_set, workers) = + start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let ctx = ctx.with_distributed_metrics_collection(true)?; + ctx.sql(&format!( + "SET datafusion.execution.target_partitions = {TARGET_PARTITIONS}" + )) + .await?; + register_tables(&ctx, &PathBuf::from(tpch_spec.dataset())).await?; + + Ok::<_, DataFusionError>((ctx, join_set, workers)) + }) + .unwrap(); + + for (i, query) in tpch::get_queries().iter().enumerate() { + if query == "q15" { + // q15 is weird because it has views, rather than complicating the code here, let's just + // skip it. + continue; + } + let ctx = ctx.clone(); + + let plan_with_metrics = rt + .block_on(async move { + let sql = tpch::get_query(query)?; + let plan = ctx.sql(&sql).await?.create_physical_plan().await?; + if !plan.is::() { + panic!("The plan was not distributed") + } + let _ = collect(Arc::clone(&plan), ctx.task_ctx()).await?; + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::PerTask).await + }) + .unwrap(); + + group.bench_function(BenchmarkId::new("generate", i), |b| { + b.iter(|| generate_plan_dump(&plan_with_metrics).unwrap()) + }); + } + + group.finish() +} + +criterion_group!(benches, plan_dump); +criterion_main!(benches); diff --git a/datafusion/distributed-benchmarks/src/results.rs b/datafusion/distributed-benchmarks/src/results.rs index b9e926dbb..3d2a8a42a 100644 --- a/datafusion/distributed-benchmarks/src/results.rs +++ b/datafusion/distributed-benchmarks/src/results.rs @@ -101,13 +101,13 @@ impl BenchmarkRun { "=== Comparing {} results from branch '{}' [prev] with '{}' [new] ===", self.dataset, previous.branch, self.branch ); - println!("{header}"); + eprintln!("{header}"); // Print machine information - println!("os: {}", std::env::consts::OS); - println!("arch: {}", std::env::consts::ARCH); - println!("cpu cores: {}", get_available_parallelism()); - println!("workers: {} -> {}", previous.workers, self.workers); - println!("{}", "=".repeat(header.len())); + eprintln!("os: {}", std::env::consts::OS); + eprintln!("arch: {}", std::env::consts::ARCH); + eprintln!("cpu cores: {}", get_available_parallelism()); + eprintln!("workers: {} -> {}", previous.workers, self.workers); + eprintln!("{}", "=".repeat(header.len())); for query in self.results.iter() { let Some(prev_query) = previous.results.iter().find(|v| v.id == query.id) else { continue; @@ -244,15 +244,15 @@ impl BenchResult { let new_err = self.iterations.iter().find_map(|v| v.error.clone()); match (prev_err, new_err) { (Some(_prev_err), None) => { - println!("{}: Previously failed, but now succeeded 🟠", self.id); + eprintln!("{}: Previously failed, but now succeeded 🟠", self.id); return; } (None, Some(_new_err)) => { - println!("{}: Previously succeeded, but now failed ❌", self.id); + eprintln!("{}: Previously succeeded, but now failed ❌", self.id); return; } (Some(_prev_err), Some(_new_err)) => { - println!("{}: Previously failed, and now also failed ❌", self.id); + eprintln!("{}: Previously failed, and now also failed ❌", self.id); return; } (None, None) => {} @@ -267,7 +267,7 @@ impl BenchResult { let f = p50 as f64 / p50_prev as f64; (f, "slower", if f > 1.2 { "❌" } else { "✖" }) }; - println!( + eprintln!( "{:>8}: prev={p50_prev:>4} ms, new={p50:>4} ms, diff={f:.2} {tag} {emoji}", self.id ); @@ -324,7 +324,7 @@ pub fn print_comparison_total(base: &[BenchResult], new: &[BenchResult]) { let f = total_new as f64 / total_prev.max(1) as f64; (f, "slower", if f > 1.2 { "❌" } else { "✖" }) }; - println!( + eprintln!( "{:>8}: prev={total_prev} ms, new={total_new} ms, diff={f:.2} {tag} {emoji}", "TOTAL" ); @@ -336,9 +336,9 @@ pub fn print_comparison_total(base: &[BenchResult], new: &[BenchResult]) { fn print_q_error_comparison(label: &str, prev: Vec, new: Vec) { match (median(prev), median(new)) { - (Some(prev), Some(new)) => println!("{label:>8}: prev={prev:.2}x, new={new:.2}x"), - (Some(prev), None) => println!("{label:>8}: prev={prev:.2}x, new=n/a"), - (None, Some(new)) => println!("{label:>8}: prev=n/a, new={new:.2}x"), + (Some(prev), Some(new)) => eprintln!("{label:>8}: prev={prev:.2}x, new={new:.2}x"), + (Some(prev), None) => eprintln!("{label:>8}: prev={prev:.2}x, new=n/a"), + (None, Some(new)) => eprintln!("{label:>8}: prev=n/a, new={new:.2}x"), (None, None) => {} } } diff --git a/datafusion/distributed-benchmarks/src/run.rs b/datafusion/distributed-benchmarks/src/run.rs index 905343b0d..1a78d4f21 100644 --- a/datafusion/distributed-benchmarks/src/run.rs +++ b/datafusion/distributed-benchmarks/src/run.rs @@ -36,7 +36,8 @@ use datafusion_distributed::test_utils::work_unit_file_scan::{ }; use datafusion_distributed::{ DistributedConfig, DistributedExt, DistributedMetricsFormat, NetworkBoundaryExt, - SessionStateBuilderExt, display_plan_ascii, rewrite_distributed_plan_with_metrics, + SessionStateBuilderExt, display_plan_ascii, generate_plan_dump, + rewrite_distributed_plan_with_metrics, }; use serde_yaml_ng::{Mapping, Value}; use std::fs; @@ -72,8 +73,8 @@ pub struct RunOpt { workers: usize, /// Number of iterations of each test run - #[structopt(short = "i", long = "iterations", default_value = "5")] - iterations: usize, + #[structopt(short = "i", long = "iterations")] + iterations: Option, /// Activate debug mode to see more details #[structopt(short, long)] @@ -230,8 +231,8 @@ impl RunOpt { backend.register_tables(&ctx, &self.get_path()?).await?; let dataset_suite = dataset_suite(&self.dataset); - println!("Running benchmarks with the following options: {self:?}"); - println!("Config: {config_entries:?}"); + eprintln!("Running benchmarks with the following options: {self:?}"); + eprintln!("Config: {config_entries:?}"); let mut benchmark_run = BenchmarkRun::new(self.dataset.clone(), local_workers.ports.len()); for (id, sql) in queries_for_dataset(dataset_suite)? { @@ -264,8 +265,15 @@ impl RunOpt { dataset: self.dataset.clone(), iterations: vec![], }; + let mut iterations = 5; // Default to 5 iterations. + if self.debug { + iterations = 1; + } + if let Some(value) = self.iterations { + iterations = value + } - 'outer: for i in 0..self.iterations { + 'outer: for i in 0..iterations { let start = Instant::now(); for query in sql.split(";").map(|v| v.trim()) { @@ -295,20 +303,21 @@ impl RunOpt { false => None, }; if let Some(q_error) = stats_q_error { - println!( + eprintln!( "Query {id} iteration {i} took {ms:.1} ms, stats q-error P50 {:.2}x, P95 {:.2}x and returned {row_count} rows", q_error.p50, q_error.p95 ); } else { - println!( + eprintln!( "Query {id} iteration {i} took {ms:.1} ms and returned {row_count} rows" ); } if self.debug { - println!( + eprintln!( "=== Physical plan with metrics ===\n{}\n", display_plan_ascii(physical_plan.as_ref(), true) ); + println!("{}", generate_plan_dump(&physical_plan)?); } bench_query.iterations.push(QueryIter { @@ -321,7 +330,7 @@ impl RunOpt { }); } Err(err) => { - println!("Query {id} iteration {i} failed: {err}"); + eprintln!("Query {id} iteration {i} failed: {err}"); bench_query.iterations.push(QueryIter { elapsed: Duration::from_millis(0), row_count: 0, @@ -335,7 +344,7 @@ impl RunOpt { } } } - println!("Query {id} p50 time: {} ms", bench_query.p50()); + eprintln!("Query {id} p50 time: {} ms", bench_query.p50()); Ok(bench_query) } diff --git a/datafusion/distributed-cli/Cargo.toml b/datafusion/distributed-cli/Cargo.toml index 0f2206794..4084915df 100644 --- a/datafusion/distributed-cli/Cargo.toml +++ b/datafusion/distributed-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "datafusion-distributed-cli" version = "0.1.0" edition = "2024" -description = "DataFusion's CLI with distributed execution capabilities" +description = "DataFusion's CLI with distributed execution capabilities, and a viewer for DataFusion Distributed plan dumps" license = "Apache-2.0" documentation = "https://datafusion-contrib.github.io/datafusion-distributed/" repository = "https://github.com/datafusion-contrib/datafusion-distributed" @@ -14,6 +14,7 @@ datafusion-distributed = { path = "../distributed", version = "4.0.0", features datafusion-cli = { version = "55", default-features = false } tokio = { version = "1.48", features = ["full"] } clap = { version = "4", features = ["derive"] } +prost = { version = "0.14", features = ["no-recursion-limit"]} env_logger = "0.11" dirs = "6" diff --git a/datafusion/distributed-cli/README.md b/datafusion/distributed-cli/README.md index 99bca2889..81ac16479 100644 --- a/datafusion/distributed-cli/README.md +++ b/datafusion/distributed-cli/README.md @@ -39,3 +39,59 @@ SET distributed.files_per_task = 1; The usage is exactly the same as the original CLI: https://datafusion.apache.org/user-guide/cli/usage.html + +## Displaying plan dumps + +`datafusion_distributed::generate_plan_dump` encodes an executed plan, with its metrics, in a +string that can be logged. `dfd plan` displays it as the tree of stages: + +```bash +dfd plan +``` + +and `--stage` displays the nodes of one stage with their metrics, where stage 0 is the head of the +plan: + +```bash +dfd plan --stage 2 +``` + +Metrics are added up across all tasks, like `output_rows=17`. `--verbose` displays them per task +instead, like `output_rows={0:10, 1:7}`, which is longer but shows how the work was spread: + +```bash +dfd plan --stage 2 --verbose +``` + +`--metric` displays only the metrics whose name contains a text, and can be repeated to display the +metrics matching any of them, like the bytes and rows of every node: + +```bash +dfd plan --stage 2 --metric bytes --metric rows +``` + +Like `cargo test -- --exact` does with test names, `--exact` matches whole metric names instead, so +`output_bytes` does not display `estimated_output_bytes` too: + +```bash +dfd plan --metric output_bytes --exact +``` + +Besides their own metrics, stages show the `elapsed_compute` of all their nodes, and, when their +output was estimated for choosing task counts dynamically, `output_bytes_q_error`: the ratio between +the estimated and the actual bytes they returned, the larger over the smaller. + +The argument can also be the path of a file containing the plan dump: + +```bash +dfd plan plan_dump.txt --stage 2 +``` + +Without an argument, the plan dump is read from stdin: + +```bash +pbpaste | dfd plan --stage 2 +``` + +The output is colored when printed to a terminal, unless `NO_COLOR` is set. `FORCE_COLOR=1` keeps +the colors through a pipe, e.g. into `less -R`. diff --git a/datafusion/distributed-cli/src/main.rs b/datafusion/distributed-cli/src/main.rs index 7b51224e9..0d4cdc57d 100644 --- a/datafusion/distributed-cli/src/main.rs +++ b/datafusion/distributed-cli/src/main.rs @@ -1,6 +1,7 @@ use clap::{Parser, Subcommand}; use std::process::ExitCode; +mod plan; mod query; #[derive(Debug, Parser, PartialEq)] @@ -15,6 +16,8 @@ enum Command { /// Run queries in an interactive shell, or from commands and files, like datafusion-cli does, /// but distributed across in-memory workers. Query(query::Args), + /// Display a plan dump produced by `datafusion_distributed::generate_plan_dump`. + Plan(plan::Args), } #[tokio::main] @@ -23,6 +26,7 @@ pub async fn main() -> ExitCode { env_logger::init(); let result = match Args::parse().command { Command::Query(args) => query::run(args).await, + Command::Plan(args) => plan::run(args), }; if let Err(e) = result { println!("Error: {e}"); diff --git a/datafusion/distributed-cli/src/plan.rs b/datafusion/distributed-cli/src/plan.rs new file mode 100644 index 000000000..9382c5006 --- /dev/null +++ b/datafusion/distributed-cli/src/plan.rs @@ -0,0 +1,74 @@ +use datafusion::common::{exec_datafusion_err, exec_err}; +use datafusion::error::Result; +use datafusion_distributed::{DisplayPlanDumpOptions, DistributedMetricsFormat, display_plan_dump}; +use std::io::{self, IsTerminal}; +use std::path::Path; +use std::{env, fs}; + +#[derive(Debug, clap::Args, PartialEq)] +pub(super) struct Args { + /// The plan dump, as produced by `datafusion_distributed::generate_plan_dump`, or the path of a + /// file containing it. If not given, it is read from stdin. + dump: Option, + + /// Display the nodes of this stage with their metrics instead of the tree of stages. Stage 0 is + /// the head of the plan, run by the coordinator. + #[clap(short, long)] + stage: Option, + + /// Enable verbose display mode. In this mode, metrics are displayed per-task, rather than + /// globally. + #[clap(short, long)] + verbose: bool, + + /// Display only the metrics whose name contains this text. Repeat it to display the metrics + /// matching any of them, like `-m bytes -m rows`. + #[clap(short, long = "metric", value_name = "TEXT")] + metrics: Vec, + + /// Match the `--metric` filters with whole metric names, like `cargo test -- --exact` does with + /// test names, so `-m output_bytes --exact` leaves out `estimated_output_bytes`. + #[clap(long)] + exact: bool, +} + +/// Displays a plan dump. +pub(super) fn run(args: Args) -> Result<()> { + let dump = match args.dump { + // Plan dumps are base64, so arguments with other characters, like `.`, are paths too. + Some(path) if Path::new(&path).is_file() || !is_base64(&path) => fs::read_to_string(&path) + .map_err(|err| exec_datafusion_err!("Failed to read the plan dump in {path}: {err}"))?, + Some(dump) => dump, + None if io::stdin().is_terminal() => { + return exec_err!("Pass a plan dump as an argument, or pipe it through stdin"); + } + None => io::read_to_string(io::stdin())?, + }; + let mut opts = DisplayPlanDumpOptions::default().with_color(color_enabled()); + if let Some(stage) = args.stage { + opts = opts.with_stage_id(stage); + } + + opts = opts.with_metrics_format(match args.verbose { + true => DistributedMetricsFormat::PerTask, + false => DistributedMetricsFormat::Aggregated, + }); + for filter in args.metrics { + opts = opts.with_metric_filter(filter); + } + opts = opts.with_exact_metric_filters(args.exact); + print!("{}", display_plan_dump(dump, opts)?); + Ok(()) +} + +fn is_base64(text: &str) -> bool { + (text.trim().bytes()).all(|byte| byte.is_ascii_alphanumeric() || b"+/=".contains(&byte)) +} + +/// Output is colored when printed to a terminal, unless `NO_COLOR` is set. `FORCE_COLOR` keeps +/// colors when piping the output, e.g. into `less -R`. +fn color_enabled() -> bool { + let force = env::var("FORCE_COLOR").is_ok_and(|value| !value.is_empty() && value != "0"); + let disable = env::var_os("NO_COLOR").is_some_and(|value| !value.is_empty()); + force || (!disable && io::stdout().is_terminal()) +} diff --git a/datafusion/distributed/Cargo.toml b/datafusion/distributed/Cargo.toml index 3a999f80d..1e5135056 100644 --- a/datafusion/distributed/Cargo.toml +++ b/datafusion/distributed/Cargo.toml @@ -71,6 +71,8 @@ uuid = "1.19" delegate = "0.13.4" dashmap = "6.0.1" prost = "0.14.1" +base64 = "0.22.1" +zstd = "0.13" rand = "0.9" bytes = "1.11" pin-project = "1.1.10" diff --git a/datafusion/distributed/src/coordinator/mod.rs b/datafusion/distributed/src/coordinator/mod.rs index a6990579f..53f931674 100644 --- a/datafusion/distributed/src/coordinator/mod.rs +++ b/datafusion/distributed/src/coordinator/mod.rs @@ -9,4 +9,5 @@ mod store; pub use distributed::DistributedExec; pub(crate) use dynamic_filter_registry::DynamicFilterRegistry; +pub(crate) use prepare_dynamic_plan::ESTIMATED_OUTPUT_BYTES_METRIC; pub(crate) use store::Store; diff --git a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs index f29fa8d01..033855e70 100644 --- a/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs +++ b/datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs @@ -30,6 +30,9 @@ use std::num::NonZeroUsize; use std::sync::Arc; use tokio_stream::wrappers::UnboundedReceiverStream; +/// Stage metric with the bytes a stage is estimated to return, from sampling its output. +pub(crate) const ESTIMATED_OUTPUT_BYTES_METRIC: &str = "estimated_output_bytes"; + pub(super) async fn prepare_dynamic_plan( query_coordinator: &Arc, base_plan: &Arc, @@ -336,7 +339,7 @@ async fn gather_runtime_statistics( let total_byte_size: usize = per_col_byte_size.iter().sum(); new_metrics.push(BytesCounterMetric::new_metric( - "estimated_output_bytes", + ESTIMATED_OUTPUT_BYTES_METRIC, total_byte_size, )); diff --git a/datafusion/distributed/src/lib.rs b/datafusion/distributed/src/lib.rs index dbdc91ea6..d4d7a401a 100644 --- a/datafusion/distributed/src/lib.rs +++ b/datafusion/distributed/src/lib.rs @@ -39,9 +39,10 @@ pub use execution_plans::{ }; pub use metrics::{ AvgLatencyMetric, BytesCounterMetric, BytesMetricExt, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, - DistributedMetricsFormat, FirstLatencyMetric, GaugeMetricExt, LatencyMetricExt, MaxGaugeMetric, - MaxLatencyMetric, MinLatencyMetric, P50LatencyMetric, P75LatencyMetric, P95LatencyMetric, - P99LatencyMetric, rewrite_distributed_plan_with_metrics, + DisplayPlanDumpOptions, DistributedMetricsFormat, FirstLatencyMetric, GaugeMetricExt, + LatencyMetricExt, MaxGaugeMetric, MaxLatencyMetric, MinLatencyMetric, P50LatencyMetric, + P75LatencyMetric, P95LatencyMetric, P99LatencyMetric, display_plan_dump, generate_plan_dump, + rewrite_distributed_plan_with_metrics, }; pub use protocol::LocalWorkerContext; diff --git a/datafusion/distributed/src/metrics/generated/metrics.rs b/datafusion/distributed/src/metrics/generated/metrics.rs index 9fd76a553..5840ecf81 100644 --- a/datafusion/distributed/src/metrics/generated/metrics.rs +++ b/datafusion/distributed/src/metrics/generated/metrics.rs @@ -232,3 +232,46 @@ pub struct MaxGauge { #[prost(uint64, tag = "2")] pub value: u64, } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StageDump { + #[prost(uint64, tag = "1")] + pub stage_id: u64, + #[prost(message, repeated, tag = "2")] + pub metrics: ::prost::alloc::vec::Vec, + #[prost(message, optional, boxed, tag = "3")] + pub plan_dump: ::core::option::Option<::prost::alloc::boxed::Box>, + /// The number of tasks the stage ran with. + #[prost(uint64, tag = "4")] + pub tasks: u64, + /// The number of partitions the stage spans across all its tasks. + #[prost(uint64, tag = "5")] + pub partitions: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PlanDump { + #[prost(string, tag = "1")] + pub display: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub metrics: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub children: ::prost::alloc::vec::Vec, + /// For network boundaries, the stage they read from. Its plan is not part of `children`. + #[prost(message, optional, boxed, tag = "4")] + pub input_stage: ::core::option::Option<::prost::alloc::boxed::Box>, +} +/// One metric of a node with its values for every task, stored once instead of as one `Metric` per +/// task, which makes dumps much smaller. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetricColumn { + /// The metric's kind and name, with its values left unset. + #[prost(message, optional, tag = "1")] + pub metric: ::core::option::Option, + /// The task each group of values belongs to. Metrics that do not belong to a task are dumped as + /// task 0's. + #[prost(uint64, repeated, tag = "2")] + pub tasks: ::prost::alloc::vec::Vec, + /// The values of the metric, as many per task as the metric has (e.g. `pruned` and `matched` for + /// pruning metrics), one task after the other. + #[prost(uint64, repeated, tag = "3")] + pub values: ::prost::alloc::vec::Vec, +} diff --git a/datafusion/distributed/src/metrics/metrics.proto b/datafusion/distributed/src/metrics/metrics.proto index 7297ca99c..53d602a90 100644 --- a/datafusion/distributed/src/metrics/metrics.proto +++ b/datafusion/distributed/src/metrics/metrics.proto @@ -155,3 +155,34 @@ message MaxGauge { string name = 1; uint64 value = 2; } + +message StageDump { + uint64 stage_id = 1; + repeated MetricColumn metrics = 2; + PlanDump plan_dump = 3; + // The number of tasks the stage ran with. + uint64 tasks = 4; + // The number of partitions the stage spans across all its tasks. + uint64 partitions = 5; +} + +message PlanDump { + string display = 1; + repeated MetricColumn metrics = 2; + repeated PlanDump children = 3; + // For network boundaries, the stage they read from. Its plan is not part of `children`. + StageDump input_stage = 4; +} + +// One metric of a node with its values for every task, stored once instead of as one `Metric` per +// task, which makes dumps much smaller. +message MetricColumn { + // The metric's kind and name, with its values left unset. + Metric metric = 1; + // The task each group of values belongs to. Metrics that do not belong to a task are dumped as + // task 0's. + repeated uint64 tasks = 2; + // The values of the metric, as many per task as the metric has (e.g. `pruned` and `matched` for + // pruning metrics), one task after the other. + repeated uint64 values = 3; +} diff --git a/datafusion/distributed/src/metrics/mod.rs b/datafusion/distributed/src/metrics/mod.rs index ffaf8d60b..94d549059 100644 --- a/datafusion/distributed/src/metrics/mod.rs +++ b/datafusion/distributed/src/metrics/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod generated; mod latency_metric; mod max_gauge_metric; mod metrics_proto; +mod plan_dump; mod task_metrics_collector; mod task_metrics_rewriter; @@ -13,6 +14,7 @@ pub use latency_metric::{ }; pub use max_gauge_metric::{GaugeMetricExt, MaxGaugeMetric}; pub(crate) use metrics_proto::{df_metrics_set_to_proto, metrics_set_proto_to_df}; +pub use plan_dump::{DisplayPlanDumpOptions, display_plan_dump, generate_plan_dump}; pub(crate) use task_metrics_collector::collect_plan_metrics; pub use task_metrics_rewriter::{DistributedMetricsFormat, rewrite_distributed_plan_with_metrics}; /// Label used to annotate metrics in execution plan nodes with the task in which they were executed. diff --git a/datafusion/distributed/src/metrics/plan_dump/derived_metrics.rs b/datafusion/distributed/src/metrics/plan_dump/derived_metrics.rs new file mode 100644 index 000000000..7f3566338 --- /dev/null +++ b/datafusion/distributed/src/metrics/plan_dump/derived_metrics.rs @@ -0,0 +1,180 @@ +//! Metrics that no node reports, but that are derived from the ones in a plan dump when it is +//! displayed, like the CPU time of a whole stage. + +use crate::coordinator::ESTIMATED_OUTPUT_BYTES_METRIC; +use crate::metrics::plan_dump::plan_dump_display::{DecodedPlan, DecodedStage}; +use datafusion::physical_expr_common::metrics::{CustomMetricValue, MetricValue}; +use datafusion::physical_plan::metrics::{Metric, MetricsSet}; +use std::any::Any; +use std::fmt::{Display, Formatter}; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering::Relaxed; + +/// Derives metrics of a stage from the metrics of the stage, of its nodes and of the network +/// boundary reading it, if any. The metrics it returns are displayed along with the stage's own, +/// and there are none when the metrics they are derived from are missing. +type StageDerivedMetric = fn(&DecodedStage, Option<&MetricsSet>) -> Vec>; + +/// The metrics derived for every stage. +const STAGE_DERIVED_METRICS: &[StageDerivedMetric] = &[ + stage_elapsed_compute, + stage_output_rows, + stage_output_bytes, + output_bytes_q_error, +]; + +/// The metrics derived for `stage`, read by a network boundary with `boundary_metrics`. +pub(super) fn stage_derived_metrics( + stage: &DecodedStage, + boundary_metrics: Option<&MetricsSet>, +) -> Vec> { + (STAGE_DERIVED_METRICS.iter()) + .flat_map(|derive| derive(stage, boundary_metrics)) + .collect() +} + +/// `elapsed_compute`: the CPU time of a stage, which is that of all the nodes it runs. The nodes' +/// metrics keep their task, so they are added up per task when displayed. +fn stage_elapsed_compute(stage: &DecodedStage, _: Option<&MetricsSet>) -> Vec> { + (stage.plan.iter().flat_map(stage_plans)) + .flat_map(|plan| plan.metrics.iter()) + .filter(|metric| matches!(metric.value(), MetricValue::ElapsedCompute(_))) + .cloned() + .collect() +} + +/// `output_rows`: the rows a stage returned. See [stage_output]. +fn stage_output_rows( + stage: &DecodedStage, + boundary_metrics: Option<&MetricsSet>, +) -> Vec> { + stage_output(stage, boundary_metrics, "output_rows") +} + +/// `output_bytes`: the bytes a stage returned. See [stage_output]. +fn stage_output_bytes( + stage: &DecodedStage, + boundary_metrics: Option<&MetricsSet>, +) -> Vec> { + stage_output(stage, boundary_metrics, "output_bytes") +} + +/// The `name` metrics of what a stage returned, added up across tasks. For a stage read by +/// another one, they are the ones of the network boundary reading it, which are what crossed the +/// network: the stage's own root, like the `RepartitionExec` of a shuffle, can report many more +/// bytes, as the batches it produces may be slices of bigger ones. They are labeled by the tasks +/// of the stage reading it, so they are only displayed in total. The head stage returns what its +/// root returns. +fn stage_output( + stage: &DecodedStage, + boundary_metrics: Option<&MetricsSet>, + name: &str, +) -> Vec> { + let root_metrics = stage.plan.as_ref().map(|plan| &plan.metrics); + let Some(metrics) = boundary_metrics.or(root_metrics) else { + return vec![]; + }; + (metrics.iter()) + .filter(|metric| metric.value().name() == name) + .map(|metric| Arc::new(Metric::new(metric.value().clone(), None))) + .collect() +} + +/// `output_bytes_q_error`: how far the bytes a stage was estimated to return are from the bytes it +/// returned, as the ratio of the larger to the smaller, so over and underestimates by the same +/// factor are equally bad. Only stages whose output is estimated, which happens with dynamic task +/// counts, have it. +/// The bytes a stage returned are the ones of [stage_output_bytes]. +fn output_bytes_q_error( + stage: &DecodedStage, + boundary_metrics: Option<&MetricsSet>, +) -> Vec> { + let Some(estimated) = metric_total(&stage.metrics, ESTIMATED_OUTPUT_BYTES_METRIC) else { + return vec![]; + }; + let output_bytes = stage_output_bytes(stage, boundary_metrics); + if output_bytes.is_empty() { + return vec![]; + } + let actual: usize = (output_bytes.iter()) + .map(|metric| metric.value().as_usize()) + .sum(); + let (estimated, actual) = (estimated.max(1) as f64, actual.max(1) as f64); + let q_error = (estimated / actual).max(actual / estimated); + vec![QErrorMetric::new_metric("output_bytes_q_error", q_error)] +} + +/// The nodes a stage runs, which do not include the ones of the stages it reads from. +fn stage_plans(plan: &DecodedPlan) -> Vec<&DecodedPlan> { + let mut plans = vec![plan]; + for child in &plan.children { + plans.extend(stage_plans(child)); + } + plans +} + +fn metric_total(metrics: &MetricsSet, name: &str) -> Option { + let total = metrics.sum(|metric| metric.value().name() == name)?; + Some(total.as_usize()) +} + +/// A factor by which an estimate is off, displayed like `2.00`. Aggregating several keeps the +/// worst one. +#[derive(Debug, Default)] +struct QErrorMetric { + /// The bits of the `f64` factor. + bits: AtomicU64, +} + +impl QErrorMetric { + fn new_metric(name: &'static str, q_error: f64) -> Arc { + let value = QErrorMetric { + bits: AtomicU64::new(q_error.to_bits()), + }; + let value = MetricValue::Custom { + name: name.into(), + value: Arc::new(value), + }; + Arc::new(Metric::new(value, None)) + } + + fn value(&self) -> f64 { + f64::from_bits(self.bits.load(Relaxed)) + } +} + +impl Display for QErrorMetric { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{:.2}", self.value()) + } +} + +impl CustomMetricValue for QErrorMetric { + fn new_empty(&self) -> Arc { + Arc::new(QErrorMetric::default()) + } + + fn aggregate(&self, other: Arc) { + let Some(other) = other.as_any().downcast_ref::() else { + return; + }; + let worst = self.value().max(other.value()); + self.bits.store(worst.to_bits(), Relaxed); + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn as_usize(&self) -> usize { + self.value() as usize + } + + fn is_eq(&self, other: &Arc) -> bool { + let Some(other) = other.as_any().downcast_ref::() else { + return false; + }; + other.value() == self.value() + } +} diff --git a/datafusion/distributed/src/metrics/plan_dump/mod.rs b/datafusion/distributed/src/metrics/plan_dump/mod.rs new file mode 100644 index 000000000..46b2237d8 --- /dev/null +++ b/datafusion/distributed/src/metrics/plan_dump/mod.rs @@ -0,0 +1,6 @@ +mod derived_metrics; +mod plan_dump_display; +mod plan_dump_generation; + +pub use plan_dump_display::{DisplayPlanDumpOptions, display_plan_dump}; +pub use plan_dump_generation::generate_plan_dump; diff --git a/datafusion/distributed/src/metrics/plan_dump/plan_dump.zdict b/datafusion/distributed/src/metrics/plan_dump/plan_dump.zdict new file mode 100644 index 0000000000000000000000000000000000000000..7249f712922cc3faa1c35b92a95ade9c1ef5f31d GIT binary patch literal 65536 zcmc${2|!a<_At(UFA4Bu2@w=i)KqOns}|g?wWU?tI@7A7b~-cGVGyD+MW8IT&isD! zMHEFvMcfcUcKZE4AmqJw&$;g|=Wb{1 zu_EZ6i^0XEzgj&dp}B6q-w!|jZ1FGPedOgbc*C`-Uy=Sm%WXe=HFMkfKMx=Kls+BS zx%u1A$5*|Hp`tfMT$(@R@QUW~sz47n*NNBC2Dhj_+0|8{n~>Vbb%v+AtGmk0HCdUg z+J7y!XP|l-{pFsXI)Cp&%Q}2LGPBoRm5a-!yLb1t`D+`mDa&*z6N1wFev$9j$OV27 zd=&6;gO65cc<~Y$UYI&L|0Lwc zG?pUSzRV4AKbp==KoV-kG!2W33mYFgHY_$YJWd~$ppOU*k4cP9kQ7kF`IPZWM~xWw zf5t>chbBah)k~msS(|35CLhjJX77DQpEyoQkipl49y6#&1N}>6f@lHn` z48xYt3SCq8pR%8;2JT2k8wOAM$E4lVhQUz8#+j6w@nR9i>c@sA#_J;_@AS!SRWx0Z z+e-Y^1T*v+RDwxpa| zO7lKL{Y?ttlRxDC`$s!u{h_@3flrG=)=fj2R@Uw6V{T$FD9la7UzvvFsZIOHj~O|9 zXhPU%y{_{JU7yc%1N8~xV&Z=KGA<@8B0MZU0hja>U93>@&{%yOP-1j^;#hrLpWyBt zbnzHFRD|dg+_`559XmP_!s365?-T4qRojrNy1H zRG+ik!}Rbtqv^o-WKv#?$Jnr6Lxm9};CdOgM|@{Bt7>t?w00_Zq^r*$|Wv?`! zGX96<$}~1IS}apjW94j=dv0KXDHT<3iZ-AAy7b78^*0A<`Y`^dWU<436|wy@R%D>+ zxr6G{<|5;vn3TyWDC6wk&71#(UoMLt+bu{$Z@N8y9=csQ@ZrXEH1BkD_Nza=S@HMn zd4ug^_v$9WzEF^1gAI9TK4-Mo<@$S6WL$iLi0_|rZUL^kr0@mD{%)A^CzzU=+(C^Q zpONDe^zoqyabeN%qkwAS6GEfI#_B`5bkL294~>t3DL(>8DK!44kgntr^gfV%WJE~! z5gH(07|X8Q;-U51JL~h=mp?l^;j;Okma~eajyp!-8Cs}92{1;@)<=;_uNfWTVj-xWkucn8+ z^-RF-9{4 z2`JcA;d!^L!eis}mFlcb#p;Ty>7Hkc%e{}ED)a_Ko25*^eD8WrA$9WfW2$UYHFfLi zHGRgFmF2DGSE-iULLwk_n zDMFhH?{&^tKOv}eS;faih7Fz5HWhVBd35#Tl+~|;G%eV5O@M~FrlC~$q@mep2GV?G zySNFk?SZgZ+G5Fzr7)JrSUO`#4O816SXitV1Rh7FBR|kLj+O&!=#L#Q2147p0;%HF0MA`^mjXU>|FUhX{7ZK)Zc?9i z$E)n^17Fkf)h)|UCHceFOJ)_|(KL(mzf3CWVp#Xd(W-A&6plJmGRl_HZq+|UnUm^} zsrU(clk^m&Uwncx9zQ{)2cIHMcL!^N@RsT`YD1Z~!PwF`$=$f6mHDa0?t~1Bm4?Uj zH|&{7rOdd1%;y)Pw25=5@;f`I@`H;h{8qb*0gtW8+U|Eg%ctsms}(h#Q=WIvUgWFk zZ4VEY0Uobue&Iv3t!zK4t{qoBH;u|$44X3eXnHza1$cA~RJL-BPx-M{DN~v1tKHMm zeKlQWrhk209BgO>tcqiarqV?7nYFO7L5ThcaBjd*i5K3d(zN@r0 z%aa*2L@x{}tNmDH^yog{gvE~u93CDTo*17H1G_T3LIsWJpbHM}+>Rf80zT}r?Y8!^89BI6U{ zB1hte{8k?}inNARbsF*2p^inv*4HNqc}0>=feJx~kux8=)W?U1!DcUlbl9T_$B-s~ z@l8TLjQ6NW*sq7eX&_u5k54xE{c1pFkAFBdG$vLb9SUR;6BkdCao&<-oV8{0`%JSW z)P(VkO^hEC5i>426rUX79f@2CpmeRcsmc;$EIFC7g>caXGP)$xoOv%62sAPr???2% zgheHW;T@~x3)?fTl2A*gd354fcnuFt4wE*p9hnc3P%B0UWeOb~7n2yvmn<}5JZyp@ z!$YCc3Hms$s%@BnY50sP$u({KT-p(3Ht+;($3`b zh8d{@&q&Yg07-0H1Jnf4ZAwjw?hW6DB{Qlc-hw8rlV(2h{ChtrbAFJ;J zQ~%G2`ULzoZW8Fzj~&!skuj|suG5c=O&G5OEPk}djVAqpU!#wT9343_O0PqK2Bb}G z!wh55*o7zxk7IpgYyv(Sg>=zCYB}6v3=8In?)s&zjrQML*J$$ez21u# zUTbvV%46p2%U8_O-ABEbUVhz3h6iQbe5tk3Skv10wj!|d^`+K^qT8(v7jCpR-dNMx zSahj1K6r1i;6bS;yQa7%yW#&+TwkgtyK4TtP*CC(0~!Qx@rWd6I(wSRJ?^^)`53Ldq4v9Wzx`iE0rwVRvzk)|zKHyIC1M)72R zv|#YLHTLyw44+nzCAXro{L|3**eFBRJ-#b&7`}gD-$oBg6H&{@)JmPsV_* z|G<9jbYby2V8;aZ{Q{o;@J;`LI!utE5s_m%;dA_WGQ@O!bl)e8>KWK-XjnpG96W8+ zPB#q0!bU32sn?9q_*)UKTh!ma`On|JiTax@fhIpOQ^n>N)U5}1QRT5&^s$0@)Q-vn zDAjbFda!XGh4X^Fm)Im?<3S4S#O$QNzDzEq3)4IaX&L|cpCV&p@wRgG=s5jocBg=6 zIy*?3biAe2!w^m&OOGWxx$(ZGXjjz25oRNL^6jzDO4HH7;hMhO#!mVMyf{hTpqp`W z1~q@rB5K9Se026&K9zF(6snrE0F~9hMz^M)M371IbX;6M6Mbk{+-Tr3MkfGM#xgVi zx}!s%Z5;Blrt^cEp;JnFEqvUz;&DI2gI=01nU?7&fYFkr5%_uGiNOBD)gW_jXe4Y- z#HRAwcLY@&p{He{EI2g%Gxv{AtI+iSfd$cu+1A)J8yUwN!YWsjFLEctR8=Vzu1bYU ziJOD0Dlp$B5>_JcsKbGxB(K&*S&Z=3{tv*^XD8dSF%kMcePg2VJoBx76in$X!ynJ`1MsE_7K!LS!vkSlu=ES( z09ZvNx54dTl9#SJ;#k0ZF3S-W(hV;`Y*xVK3Ks7OU##;8JfB!_u>GM@P$#Go)CejB z^?~X@ZEVNF?vmNmlY@t;Gt+jX#cvN%^(*#Lm4&O(k{NTUbH}qOO*dQim)LXuEPtQn z?6W+5mYXlkM=aApM$jV25I(ZU{Ssf}){^`Ae=o(Eo0>*VUwDX`bZk2^&f8AidAf&M zIwc2PGS8+aO^4+W+RMHa$N$J$bZI_1xPmU&(5QYDdN!5bSA`}3l{PHPIjR>oO4cSSm%13({{@PIf#0_7|jFvVn{M)ziZ{4B+Sq@MXOH?dfv82UP z7fWC)jj?2gDeX`4TraUxgCIX3AIJ}+IglR^5#)zuId43HlpsHl>(UakN#i zJGLXkv6=k^iUH}4#r8zK`NOAwzVYLVpYyw4{8N4Sq#-h*n^I5l*I@E3^^|EhL3DGx zsUW%umU85NXVH!Q8uB6=!p(No`c2uYNV%^lxk^9IQ`TOgjZZueUyv8sa2P~n6RQ6; zHZHUW>|x0Q4qpTNk;R)V?cvcd%i&|(uL;t{9(I}?b>h+HBCwXlBXVd`xm0@ zowHE=$tncP2g~IkWeJiT8wsrHj_VQSJ(VR?unp8wMXBNyZ3)%c1}aB!f<9oCR~H+o zdit<(vBEO1t~O9>=mUzyO4#^GO5e>I%6-k5Pm5-EyRm!_Otccf?$&_J)dR!R=lu26 zqd5q6HDovJR9Z)J8-P zWOs^WV+dqR4&b4{6CA0J!&^ESP?nE?WgG}_kqMyIff>mJM7XowKgx{9U^Z87nDWC7 zI02GD6T;qF4G+8p6$b>CvPy@9%^*FF56q`yS?y+=J~;iesIpW5w^1A zzbvoS!GcyN3tEBLPEKElox27P9*n78UOR}*NL)GEIFO#Q+5O`m|A6_`vDB1>grha^ zA+4pQB+a9~GM`F%w3%A5aT5i^YO_6Lq2|;)_t=+vq)~b9MFw6fD+XmDuViPrJzBEZ zZAFgJ&5B?R)xfXEA6CmEq_gh*K7^S}be3SX{sI>D@CcZq;v#{QN%(e&@u8rb0MqxU zeRMw%fFJOs3{=>!c)DhDz(*?f2#a}tkVkZ46lOW$Ila4`x7y&~_jbiE`y%3(u;_4o z1gr#bHrA7uSU+RSWSa++3Jyqd`kxac;h+~fIx!4p_5{5?zRz%|gEc%zL3M#vS$EKV zAN^A_$O2%@*;Jb#eAhwuL%R_=Do_U>HL&3D_M&AHKH>7}DlmE@qr>357aBeWi!-9) zwTJ?CEh{F7(~pe-wnuo39GDw$H`^qk_nDSF-w)V5z~v83jK$27c$T;Ki#|dNEU-3= zHXSoUi7<-@et5+iP{P8DP}+Pp7T#+Q3d;^LSJiN~kb<$~dj|_zNM@Wc1`hfpn@t9taq1HF;Q3E`~Ulscdz%ts4dBS2?k*k|^(JeT_M#bmi@XbnfaibVka4+O+5# z4MJjxpjIythsxhcxr2`S2gCDWCrx$I7iu@11skUhkqEb9$G~ zPwSm>U|w(S)(@GlG4ZpsH!LRhkAXay!$iSuh9yG)*!p;kNL27J+kSXLT%vx2rZJ&P z?scl(Q&IFU{{v!9yuITddgtwHnti8x?%Vg?Y;Socy$#F>>^g=eC?Khg3Jcd0CLE98 z{{1>?8|bU77nBeA+EmWm;%!`J@HQ>V@-by!QX7_>RznF02c3gadaJ!*pMRze^9T7| zelXshlHGh@Fx2+@A>YCZ0gvDUCP_Ta2lA=yU{!#W@DaE|F!=cjO|9|TX!a_o@v5u# zDt_o?RoyK|y|d0S`>Q?goN&vk>kq;-$vH;4tL3xXw94!9Yqx#(JnCvyn-BB@F0=$e zy1V5NnAZWNy2I*Q!SkC~!b&%|JnYOyYrXflFKV3c3B!PlKrY2MW%1j#Mdzn~V3<9# zLz?kopgC__pmk#s4HxZ{R?OfpINmNdMG{WoSFkQbS!@bNj~w4;xU@-O2F2`FSg|Zy zmTOX81S>L6T0AUicIoIAg}JWM=gFOwO!AI8ddc4J*DP79US>Y21Z6x2LRv8!|DCyh z3%C2Pu4uOUhGuh~-_~c1&s_GMSgI8=5G8w(Ewdzp2T31q-_+)8;b%uywkWRYR5H0! z(yi7>uRhUy?MT2YW}vHCkWODdd9Kf!0~g{Z?KwYiVOd?<$q&q`%^PyQzVUFLFDQ9B z6atG``R`Bld%Ua7@w^X;A2-`NC1A^J?c6;9t2e$66AhaM9!fG2VlfN1UY~y23qGSj zY0I&^Sj5ql=TB~)d*(^=7mp0BlP_epEPgY+<$((`8((<1qwW1EH*}g|PRNwS{64I% zJ$Dt=-CVI9mFL_U_jqDK0h&;Fx(Xdg+xq2=J8zGpWhWOLM4F+_N}t87b+0$AX;xZL z_0_qhsVyJgzS!fzo3;IxCSUYk{bWVhW#g;XnqJmC_dvt!CtHx=UglP0$b49a|M9m^ z7i>kQnZ=tB0kI>Zn@mJ>DXfU@PmV?0VnG94u$;^V3(j1y6b=6Iw^)$of+cALMDs}l z@^CRvH75WQzCcL9%F^u#$wgd$*E-7e8RfQ?W-s8{^&thrl!FnL>QjabOqrCC^)f)|z82;fB|-KkD#12@5EYrC|8 z`?%{f@Ldd_z3|bDaVVn~a|(IB@#Kf|=1dMbTRoxezLgV0-d^9_;l<|3A$zAj`4Hcy zO-gFJXkT*3&6Ejk=j==l*Uu4z3S?bKvHh##m-*VmxS zUGWJD1kv~jR_cMfe_Gsuz?n}Auu%mh`4mm7?rcDC&SSX_5^X@7?0WCcjxL4OU&Ee< z-AmbM1Y#)V^k>^W)KEIIFCtG2hP*FbusYKQC^mjVS5hw8ogQ{HBy4KxCDc@rGSlN@ zd?W((Kr1x|oG;b5CF~nnh7Tu1D-O4l{F^^&U<*lJerwIzwlz!of?&R3 zy+e%Pb}TI?g)Yj(hgv7K56OEnmgRWuo2RrK#nZBEM=@>&M|y}jH*DRBRbU;fBHBXQ z+h~c99i7=uvEWb$I5ax7d14V7yR;M#8rUj{Z)j~#GZBNR4u6(5kQBp_aw3NG#BW-?1psUy6%`X7FSm)| zio&i;zd>8_cPt99R^7BWdoX`@M$I0}=Qz;_R*1o>Ns?9w=?ZlX?hcF#mN_U(k*oe{NJWFc|5D>y22)t zS7#}h19l{(BZEVuVqxb4%n@?g0e^ucofSE<*Zeb1*2q2txzWhZ7?%hb$t8J$afw6` z9ubK=`yIi`61f|I?1FK7k@wBvVw4Od&^`g_eTa>D%X_3mI^d+ay!e+rIJh;Y^exSCRr-G z#K~3_lL{?muzN`uG;DGUrb#+Bs3ffxEaj=mrCYZf-Ze_`tsAbrRl04jtaU3=Ki z^1N=!h&!C@CP~G$Me`I)tFNPCMutUwi#6R@9xT?N>;uOfo((G%H-P#$PchJN4zj*H zSHXDpjf44+*XXlDcM}3Yxx_{sNy-Q2HSC_$QoW5$;|)-xo(Aw{4-+#KP3J)D$U-!= z5hTKM>Y)OJeKx{FoHDa1?2%;!hB(zkIcN=>rL_e=MuJ5RvXnE%XH&K7M%mNF(x#7M zzOhn%vYCakY??D(59N&01`!0R3p~&y=~m;AT4gm2SdYVYc;sN8 z4$w15XE}_fg#TBg$bQE8Bbj-~F0;FXj&QgIuD0Wz;^hhUyN8ExqDw3n5z(Zsu=EE9 z?R51Y95r)>NT8aFP0oOoBGDCo|JADc5!>f39iBF!CS+zo&U92#ksmtk?S#Jf=I$M~ z=Io}y@5<@TpPg-_UmY$xNoSQj+CW{ey}6x!x%k9GbbnFhS^8Mjo_}pi-#r%GVmP_e z7g=l{&bfK%Fg5wklPUDdibK!Px;>`-RNb~3BYLxO+YI``j!hj-PgsFD z?Px>VtUnt}kNO!3N`K|A%l5vCz}GT-UK)bbYx@x{CIc~ve-7ln%}`486ula{sQuxmag_JrES4}N`Jnv(xb zT-pd(G;ny4W?@|lXZXC+yQU3X*(+_J2@W@A;Op|&^WpiGkaywpR=$4lBtdb+?W_JQvrQe(8GwwN2B;qaWW^jF!RTP`YKj8ZGhn` z$(!T&cJ5BNs8$KUVC({#k)SjaXz@C{!N&a6ZkTb)BZ{TL$$%Q%og@z$g3BrvE+n`Y zM()w2b9V{0WjbK@HjIIF5BrrX?C(4W{dtN#p~g)%ilFaC|@s3VUB1- znkSHpg(?{@jTE8h@C@?!Fc#5>cX=kSlmY)GGE4<2oc{6U~xGldd z_;t-MhF@%4zid3T*vC{*qB*+tu+Ou-3w@qmI;gq1Wv^yg*4)ORf?@5|LU26cWkS+l zF#B-i-oh@L(|#^^w`AWq&@d9-(mHq1_Q!Nu)xKl2>D1~+bV}y(<8;cQN!RH5g}Z8~ z%K0f7*o&Hs&kG?7zY9})O zY1dYJWSrXWQNLlkhp}q6r>X97qv~Bb?xquy8T@)#jVRU)66f{DZ*uYv84JHNZ257~ z^^adK{2V+Wt;_nz?(1WT6C~ba(7kOH?%A*BxZgUx);(?83O9UTIj=-@>hVHXaNFTX zNlK`LU+KcWQ%1Yy*_-fir*|B!Igq)?*YtWJ(-J zVO?h|868Db(TJOl&{0ugqp`Ykc<9K;xP&pR0&CoOtn&=0F{5GD0p-&0Q2p32Fy;r> zBB10I9|F#FI3@lbn9pd}zejVzX%y=2mHf-raXS;sem(QcvT=LHfifLtaS6$O6Y||( zUsPS0;r`;Rdsa!KjfWbYuT_E4B9;LPP}t)Q{=v^d8Yuf5=|D5+9q5OcKVhqDmIl;vsn}OQ>urp<4XZie}kV zi-!Ukvn*iF7=_nG!)Dv#^dmVdZxB{uGXZgn3YK3ssg|1oIS4_H4|{CgN47agEy+LD zbr$T$;Z@ysY_0rNhKWZ_K&%cC(DftEoX-XKX^t%TA%=hc024ZVgK|xeRymC%cYdy|x@Vpsx zghK%+mygC^Xlp)Xu|$+ZWmqKqUqG=Z8s$-%`>{yi=b^hZGEyFo z_m$~_U2{1lEafqkjZ}?PjEZUGCtx=m*vpV7Q2Na`kZo z`Vlqg4!QyHL-eCyA26nau3t=a0!jWRGC@|niVx1bw-+HF5bO2Pb!Sgd!h68h5YFSB zgMzzs4C=wFjY(7SVVq0t!YY+nygj6QFyzXz5`+?l1a}i2VtwP_F8CBDcbXRpCq10W znh%MPg4qyP7x;ogO40>`A)y}9+^vq~tZ|YEiTMOX(TlG&r%ES6NKB3z%8Nga^Lfd& zptNkgEKj>^*(}5nOf*jVhYtAJF~N>CwYu{L_*nw zCy===IwpbKw;RC?N%m|makmSC3;OjW52@|T<8y7XJ&IjVx)`N$zZ*8Vkv-{gwl9jJ z6Ep1_$neH5cT0NP%?Ka+;AM(qg}brwo}+^uE>pn)&jzetF5dQQRec<}sjwp+8Lx#R z_f}VsR7=+v8WaW7<|%TQu2Gbi!ME{@BKP(!h3V=o1$J;E@7Pe1N!k7iWx?{x%9M+j zm4=O%m8EOSmDSVkE7RspP?c7I`K(1I(leVRlkyWe-K(=#b8Y;0IG>zD{8AZq|H zo{Sj}leOh@I0nK*Z{PmM#>e<>Wx29cQKy)#+M>FsysNO)F0u)oUl-nBD};ZvYY!o&fR-1>T66`Roy2?G-o%8SH5B4*q`g4ENtf94I3!7F0H<5^ z5P^_B!^ax7lOTy)0u~%*tOSgbqeeSc(?%dqI09kevSXZfy1`!z88%?pfI$OwU;b4$ z9P1->vZyutfUg(!1=Pu^F5EL|=||{>4EkZH?#uqee&`P_u)ZJo<)H5e_UqqIH+V?@ zzWw_R82B|=QpD$94f^)mK|jEARyW9IRE>Zc$x`FVmONNLzajxO*;kD6PoSdl^VXr+ zkq48|sa$k*F2b}YuZSa}s<$u?RAqFDYpLRiVv%Z*$`VN`-j*OOa?MgbR@BoksaNzv zu-LamlFGa#NI43l@-4Lx-nND=Qyj5Ek}9M~kd`VQD4)|tMJ~Mr9Bmx8LXv8!NRXZ> z?kjWYOw~&IJhf7>+zLsm_9H>6Rn2ua;b`t1Zk17jy_2*QNE^@Dqhkr6moLilv)Vsa9?F8%5^mu;bY=HY4B}c zCFdGQ81|d?AZDuaD`J$)YT}4KU)X`dn?qVSzh%t=rH^17vTOv&*kg6TY@f5E7*>bS zc>T|;79AeT+=CG?O<5APdrc&JkZ{5D4vaNP`(*Z+6j;*dQ14rvr}n+jZ*5P@MwMzt zNst~XkJF2lrAi|>hImOovFdX>beyGJ0kir(=)NV&({v6ARw9lVJ9NBUVW3~ohro-> zQsr{!K+uP`63a7&v;e+#w(&T8*y&lVJv4`Aj-!G4cE@6w4m;Y$_vG~CBM)SSvj5{>m_tZPPLvt9QCU3_ZG(UBk;>5W{)S>i3bbqUn z-Z^zD{kVLK;-GO8o{4PK&%_!u$=^X6C!q4syE{?ZqXq3w}1;D{qwUNcQZR97PM z>&AT}S?X`Yw~u}5*L1YiWs|&y%Ru;olBpdy3~gK!DfS=#a;S`)PMm|cXs?1XJnzLo zFal-+&)B&168g23)dZ%=?P>lr?Z?@jr(HsKf*xm~g?+Fd4rXgf-fckS*NyuooUaWQ zAVR1jZE4H@H(twxDvH7n66MLS-~fHCiYl+6vR6@a&QQj^)RR*boB`z|umr>DKr|>`;7GWjpx3|1p#I9ml=n9nih6<7*GJq zO$ct&A_)GYb=$FHtq!j%Sgs=BpSl|ih9oktfkQT_O953Hag(Y|ZED8~%#SSC91qXU zu%epGdDyL+=mo)b)$*3unjWZI#v+WvRg&cnXoH||*5YH669S>xTh&*yT(>?^F5S$`pY94` z(}o2?jW=wZrFyz?r+P=ukMkZ~54d{w9$s>tkq0$OedU_teb(!U&l9z?;vnLq5u5}# zB@WVTnNZW_$*dRj+GU}KFK<^zaOgXGT{Nsyd4$DVtBj%4LfBm)*- z3ads50)ohd2;iE4x9P*73;GNn3O*;p$MlZ|l@>h~WwG~z$Haxj>l-Sd z6gH#_k&Vd7kfdNzd4#N-BhUhx6(xZIvF8ActHun9;U0;Mf-jU5Z zmPOsXsBBP%W}J`X=Qx(3rJMrMmpvzQybNvCU3dF1oCCQ<>7$$wq1A6?8W5qao2zPd zxWXwBTFRQobkU7`MaJA+irl<1MQO%0g=vpjVL1I(QM!JjmGnz5aVRGqy>vwjYL&U0 zo+t|ryi}InUZAR;x+ftQz#&f0Sk5W0Dya+9@>~?CMtLa{5oomLGLio3Q zQJt$fC$G-6VB+#R*YY`=>s%{eEva+W-n)pHIy)A{7DJc_O?dH_bTG3C*oU!nOp~Xa zEFuzA(JY2GNt8a540K3i2faAl$s-u~;+Q8-Kph)fnWD;P=M0+`yoU6 z4;+T2Ev&6{+)pyR#)yarmxxjn5dn@yESbvQH{VN9FZWYFZgQk`nQyu$Y`Ytjjd*QA zy;70-D!N2nMOzSTyCo>{9A&p&z0~SnBjFs*0+YbJT0g_i&!vC67W~b0r6n*KcSD97W0{2jZ_6<#Cgw05_#d=+5cTeL47 zZAjUQ78G4ZXOmW;n~xWu)iu}A+4-TE^Jg7&K|u5=UPD#>^YyT@|8wUidiB4LA+l#T zd7h+z;#ag8S^uRdH6OBbs{q!-V2(AIel{3?HdMzNGJZCsEk%a1p08Nq+Z2>wGx@E&~+6g-o3_BPE3#=J39}K&J>ajv39D5St&}%g}vgj%2 z==5{cvKQ3Oi^_RL_JW>bXcZR&yWyaxQfQ}*LD&Nw!tA_sxoFWaH8%kT@l~(bA@u{bIMk7;>8%DINzat6N= zpxYTSBxYn}^q1HZSD?Jlm|#iajO2vCo=d3LKG=5zm}kH%LC^x%|CC&V83y~Dcj#ad zru4yI>>n6;Uu09(0X{>_anOc>izPiZ13Um^=osn0Z9(2kDy;I`nM9P+|# z;rUI7guQ0L4;L|5C8w6)SgT>4xQNBqxVuw)aa+7z&ho}>QK4!M`Qx@2P)H|ywb|nJ zvdx;Ja;2d!S7RymAq_<~%xHiS8dlb8jNhJ-S;< zp8<6v9$D6qLPUO5=sW*rz*Mt#**w3;CiT;qji(g82ku=R@XrP0fNjU`PAr?n@VNw^iZoxt%XHt$xn%){5nVf|xL^0r6-{8{=7iT^ z5!;0lejx-1SZO+C$RWY^anW$Uja_sCegn*W3BL9F{Yqq`^$HjPPrpeUX#Kg}7n^v1 z|A2E&Q zrcg{W^_xvUz|t)~CJIMngoqvSKXuT-1J2hRxgHsbBb|sr-pND4$Vlk=8-ou2fHxnQ zEB}YEU&8uFhe4d*4!U8H3Ha;ln3xD+^Uy&@zJ@^zPVUmiKBaFIL`6;*Ft~$mh<-G% z{fW&AXeHxsPL01W%NH6n6s%bw<%`jPz!|oHS&KdnN3R?pTDWk}`f^&&R%! zYdFgd)F*Ht_=*J(D#WFT_iTFKn3$g;^?du{wvG>vgP_U$lL7JH!B!AAI{Q6%OiZ-p z&36#`6YNDnktb{{aMuP5hL?|viTeq+%dogG2usZdoW}R8U_xHv(gBCO5t9bQe-W1u z3E0>e^#=&59y5;ZO0LnQkN9R|0WdZEjg*qTmypQN$f#fRaS%3oOv0b_aq-XvpmAsW z5ZcoAA(%wlJ_Mua!62FJJ0>hH9tX1S3ys0$jA`cE$Y>ZW1LFHZBptlP!q)?16262j zb~v=+SKmiR{*2pYP#i?EhL8wACmnR(;3os(2Sr72^oj2{avu;++F1{JidH5kxRDaZ z;MiM`LnN4BMPd5*HU`>H(f~l)xEFqfETcig8g4*$5hy%90lE%0{V@sAV9kvQ5C`YM zqlRtgfngYS5SU)-peI6u2oFdV;(!M_G$uSU zEQ+HI{`LzJgBA)rY}nxbadEiBL&5Vn^dMMxLInp7n!oFqb%%T=JQ) zH=z5=PFBOd8>Wu~h_B*e#tsXEwgXD|y6?9*D{gXPNdqX1y})S=2{}o8LcqXAauAac z2c-UUq<&mG-M0h&*kAX-=$M$%QTh*lvrv}EBb^N#MLQ>p^m1Mn>4_yTcBG7;!i^8A zbjr-mPRJt7%O+rtb@l^C)o=39zKUVy+L2)dgTJHpY&1x0U?FOfV}4$zSmI)zZF4pz z_~0Rfz8mxb9}zqX#Mi{o!pd<4WuO1e3qy+gV2L`oe;eOkPv&K8vxeXgp9R6>vF=vw4Zf7 z!Fr*Jw+s z^}Q$2NyBMWbnzsbaIp+6xm||VPloW|HY(Sv&SdsIn_BVtf$}T8Qy1rdcBJ@HzsqHd zKfSkh>nCro%>Dw-P&Q69CBn6x+l*8LFO-vMh&iYaDGKO3I-gz(Rv@|Xfm5R$xrT`3 zzxy`5_dfmRHofgGy{Lv>aErcOMIXOOJF8MnGy<`9WakEi9O65PdL%wN!1mudQlJQ> zJ2L&^mv_A(fWQbsvq_~lzh6^tW$eK8i+8xby|;Ad0no|47pG> zeE*FP3rruLUp#!;ov%Q(M%G2B?{yqlZWy~hYy-akNl)61ss9s|8f9@x=y8#%thO+N#?|I8xsZ!Gy^ZCr>Xt zk@AGf6Dv=!Jkb)u4Qohuo4kT2$KUb_?b>F{caHgucsU9jE}1a!3}2#e8vDT;A8a?m3aT{sl6vn9lWBXS60zr!hLF6^U^oD}%?I5LCj7l0zDwOVN$zFrBKb)`yTOm?%c-CAkV>qEj!7If zDvApRCc`wh7D+kZ-5;^SH`X7Ot2SVM{&RZ%_T`Fw#yrL2r5TESXD2FdJ$*_)U3V87 zK-*~1^N6nQ><`R)j&xXDZQz~Pq8E%{z_XWKah33bfa*K@+{}ySxurX~Ensc-q_dCMJ3y1aSQGCxVj_BF`DRIXY#KFYY( zU_#7HSr@jj?M;B)JLeTBl0Exl_fpt{j&KWumJQj44dCERkrzJvFOG*ka!5 zQ5V}sUE9eRYM%DL+w^SH#Z8l%8JgYlKjv?1CY)Q&a-+kvHS12M^u}I4+U(@Z9#o zbM^y|Y1OJz$!_@P->v`mzEKB8-CcsN4$9~QJ9DcE#y{gotMZ-i z=N^2&;J7YrLL0-f*5;e7sv+2ktw|FLJSI-k+`7~L6+k5q zfle@`tN~iV)G`I=r5-5etiN_%F=EouV8K?8Rfc0N@&$UfB5$%`I)zzz7h;yLurfEb zW0zbjxtdCD339{rRP#sG<5^4hm zlqHqHsmO}Dx;aBtHt{_6-TI>7Mc;~>o#*WByfnsx|G?C7<=GD(FKV&vc>CPS zR(Ec9O?uLCQ_jZ?oK3QiXr|W&oXGXvab3M7zuAi$n%hf!ubuDU#M#7DH=E)=#<`Dy zs~HJQa%~x6j-iAezme@8Zh3r$BX&G^+_U8FH^erwqoDuW`9yAkZaVmAUroV5g1&G* zpsdu_CE2S;9$@Z)geSpS=8I3TA}5x2u*Uf8wXnk(fp;CC6D@9*{ZAaCw;!f=AEKWe zrqhnlkB`uk%jo1%+Lk?Vpx`ztT6q!GzS@pT*6l;Lmgb?&sYlSUt-H~>m%D!N14)Bh zj-1V?EEk>6L6bM2nn~!)JSr&%mCr|bsU^n=Sek^Ljn$SzH~GX>bl}BlRB(4QdYQEg zojtV*T|2%PonBRp&Quj)fwj|H5A|->hhXi$ihiWXp${sa(eJqBs7*o-M&IY4Zj8Nf~Xs{he^I{nKK!WeZxq7;VTvRjW|` zMzm=A?@d?=2-iVOc>qH=WJmBV<46ECj;{gs0gulNd`dBs$Zg6wl{m6tX)GR&%UE4q5v-F)w1 zqm-rB7{lAySZPt_*ehn>e{W@{JC{05+u!NVtxnU9cY1NR)8U05-&xy9ruQ=^O+r<( zlTrHaNvQbwBxE==38k7Rp-GvOQQet|_~2t-ms9e}(!9ng^HfE}EsV21Ha_wyyWms4 ztF3WWTTpVguhJ>v>oTS;rVMx1p~^F_At;iDaHT8_df^#p)7d-&tvbtRAKCDrxO7cU zY7*L-9zN|>w>xuRqSa@PVoO|`A<02E+ibkNMYVVBokG?6`rS3kb#ugMLh^+Cb6ka#%8!m7-oJBluKOYts7d9Tk8G!z3oE~;Y5v(37Usuir zaYvh7GMAh0vb%-Kd6_B7+pBgdP8#l;W0bS1a zI@|GV-?QD1Dd1Ciwz?F1^ejMX8_#M`hU^w_m7nESo zmc~0&?ycU|-4CfEnR=gC|6}^*KfhWYyl2A?h6Uju%(PxGxO|k@6@?CZ&Cya}2@8Ie zlaN2tBns;y0Lc(RK8)M=(8PF%^UDJh4Kl36ZkHj-LHe4W9Ak=B6uTB&-Q-dFW{YdV z^(~FS6i8-0;4r-BCNl*O-FBolx<1Kc+9g-4Lnray;a$GJxp8xI^X}$`w*f`-nggBL zpYQoB>vO$Ro~X;#w=BA%sb13pwE^<77L`aBktu706!?z~Tow=}qP zL3UO2d5pYQJ_Qeb%+GuZPWzaz__TVec0&%7^wSpkSgPPljSI1cdO>lL66Q~d8o!m2 zzOww;1~bx5RWpBZZX1q;)MWmLt#(iBbW=u~A6vA-K*8evC#y9Q`_}8TyqwZy_qSPX zabA1gyazE6*`*bzz0Yu9%kX@hhEC8DP(G9r>(qngGLVSC+c(N{ZN5F~RfV3w&GXw1&=$#0fhkd0<1^XZ|0Fwi@J7E(g8cj?ahpi;= zsE5u$Y73JI?U7)+9;rmzX9yZEnqNPMC=e3E;_2BWq~r!RdkQL6A0(Jxcau)z|CRZ* zMG>~F3<85O>v|lRJ1_)*K5kqFMeGBZl>lDyze8W@c z0;rLLjE&E$Kfx+K*HKf(FPc?d1l<%+ zs|p{_@`o_ND5|fNG-QLlFH_knsiT{+VK2tH%gdCx=1OJS?#If?Z7-DNm6b~KqQ$EE z>pN9t>#nKbz#y3erL&}XT4W)hcApFR4Q1duQK71uHLBp@P?Qs!;vZC(KG757qo}sgjrPFH{9iVD z?WL?}!f-zwetktY7$0&7Wlq$3%kb z3FXG2DWU8*I3<)Hho^)(-~g3S5Bw17f*(SCaF|M{6Mm?z{?Ly5GaFSaoIggvnc!^s zYn&UtwqSk1|Ck4}XWW6oXap5@!jyw#iEUq;GTb20 zY5}?dGYu|pFbI`5%7AnQF}nzKXm$Z$u5YgPS$6&v>{BS__;0tLkXTL#f&^kRNJNC7 ztMhXr)*crFV+f)L5&$D8sAEuPJ^+T01XECFII4CG>IzA6n?UZZl9&*(`!)_t3_?jV zwR&HJyFWe`h|oYbe$t}Z*#M$Io%UiEW{n(o=Ut_#k03(iO|P~ zW35zhCPH+(xro_VGZkmdnZ01%4|`UiT#%OX_*|nVhiH7KDmmEt#-Xurvb16CkmPme zd~endf@C>%4u+uH8xc4Nu*1Y5FH{oeHA&dKUCQO*oIPp?3~bn7UP?mEn5OpFH3SYu zbASz_2B)YpwrQvA9)l}l?LUG|p6q_^jRHz4t2&866lO28m&S+P^AQ|_;eCUl6`rf` z1{pu1pq%g7GFS+@hsWo5ZztKx2Lmt*PCuwYzPX^b<6jm(WiNV@Fev##=0$ zqQJAj)2e9NTg{j-jtA3Ox_P%U4TpitD%pV|2dGB#M&W%hF9wwRN9&2>G@^zV-=_g8aO~Tyk^$Y~^;XFkTUvIjWSg2>r3>xG7XLSBO*qo1_^oLmown~rQEr!E zNiPlV6D+8U#6~8EEX!^ZN$6eklWafaDDqvi122ng@hs@zzb~F0n*XiE`<+YYU{py< zXcbcnA_qZCcgsOYI+%nkv+Q8WP)gv-!(Fw?`X$Qv}%+xrK2FNPv|N}P28QOmUCPKAfD zGRyI-Hm-1sb#k!VPNKX^F7XrM){`iPITHs~(j8czlL(QqcC@q@QRA^QQhh|=a5%Ab z9?=fARG3*U82EcVR_{^Qs5h(2)m7>TYN)z}LovU~MIlqnA(t0_b2ItNiBI}nJMn3D zUC8ei54o$Xa=K7BKht(pu^XkGrsq9IRXb3|I+S}KfymB|mY9gRY=`3fe#M)!igo)H zrV2%Ij^gkR1$a9pz^k~;+a5PVj`iB#DZA714&d;=K}~cKxBmNGVP>gi z9`)6ms79S%gEp%-0&`UjT7OmQYc=SB`t@C;tq5hZzr2;;+UbJm#Pje+QB za=XY78+~FfQn7?W@zIv|Tgwwa4(lz958zw^@kwdZ4=M#!f(k)(pfXSus0avNEl4yV z|HuOR(S^PA=1niC`~~ailZVRb)vMOgg*z5wKkPQgW1-pn4$9`wdwgQgPW)k4z~Vz% zV=|OifDl0aUM-;ZWKg^3Q`hEGS#zmd^Ql5Or8wuok?(%qd6K=;weU8RUtq-|?4ds(Rii?NFgd#*{N2MaxZ%P0#DyOD8S&1a`FyLx3-n zX4$;=YZv=%C{y3t)ih^Ci#2u4Y$xAU*SqFE=z4xv*VO%8Yo>Iabf9a_y{-_?nPnWX zo+h!88n1+}P4yEzVOn(Lf5V!wad{mK9fRjSQ3{~GPg(w2EX`0tzx;d3x(a&6BWm3x zYSm0-ZknPHoKMTx8Ino?h7f(H0Xst*c;~e)iRc214Z)hgK97eTP*9)%bIo~Qb8U{B zF?FJM^@-^%N=r9-rByBTDX-b@34|!q*8;BB#wKZ+%}v(Fj?8> z4hi`-t(>(0l@|9jm&a6_&!cM7KkKiJFFo?3=5q%;BLQ1m4udTBMmDNm`*Z1JGb-CX zw)9vs$~ZC|o7&1VoiH;=$iV+f@gg*PO3#%KW2@-tA zZ+7q2QZzlI7og`CKIGA-oJ~a_OmZZ07IVNgFr4ZM6{xR!fGX3(3V_bS9n_fd8A$^9 zB!I){cyLw(W^v;Cp;!1m!QZV;csW$iF}}evZabo9EHtS%Ec390uu{-b6kVA!RC~ko zH+j(nm1r4*;Lbg@b3B>uQW34NGba)dBKr` zX3}%gNsDMrkVbDUS3s(~qe?d4=G>0R#yKUMwMDYjAtjsqKGQ4-HDP>X6XVB_7>L-^ z3wVf$pm0khpe``Ev1^kh$XM*TOMq$9K=38}GH>8PH|$`NEmnJ{oJz8=9i4R~$$~>s z9XgK`3gCppNEY^e`%WSOfIc_9DYT&jXOI9y&A`GC76v;E0=G+%V3Vc)1#~HiEe_78 zGEXG{5$jxiR9Gx1SB7Gjq;Lks{Y78c{ufw(9mYYzV7HU8_K-s-jR1y%K*HXj{9AG0 z@BE=4>4(#Mx_}-QTLRg!DQ3`S;grH7 z=+?z5*Zmte8PV!PSL@KxiCI3oYr!4Q<)snnYcJO$*vmN_!(w*+hbK1Pbkb|~6W_Nx z+dW+zu&JK8@HFuLL4P=gI~?C)7H*$^{%QZuKm9TaPGgQoyNIK!sGr?AEpua+S+(0f zJiY78XQ!5I?>aRluVqp7p^zuj&V8&I=7dI!nBRxj_8%xiSNA_%k6s)&npnEbFq>Mt zZqG7w_O&_cRQ2h})cPC~xKSPItnQ4MweIEo$J&Y)%b7`KQ$1gn-u!k;@>V7_XN5BP z_?}@)%91rQ)Pv&^O;VwdPNb zHFLxQ2D)ICC>N~zKc;6D-Hk9mm;(yYQlN^&&r`dwJg(0^doa|6Dd3{LXq=zkY9_EMKE#BJf z*5$&GSE*Y*xV-m3kIab=Th{F@4EZnPx?xF;g;Q5}AUJi!iQrU&_4aBh)8UlYo#OK= zoOj2E)yo>-^D8oH-e6@ldtiKQu+rl7v_pSX&%0>d3qZ1K|y=ef|ePC=3;+G;j`+ z;c!4Il{^^A%l91xu^1XsVNaRmhGAqG1RRml^-l)ah2b>V>56MD?8C4M?DUlj-(U%u zWBqly^i?a+*|Ntlz?C#7w3^Tg*aJbbK{hy(pTtNu|}!Q5er{R)Bet z9hL!y=Pkawbxq5XY>n?Hc}=1;VW@(SE_x~Y8}iV91n>wW?*5Djfza*yMVLW6{c43 zye!%6_?yo(PucuI?v{@$AGS_g>J2f6GVXRPI-qsPe--yQ&CF~6Q?xnWu;nLXS`sSV z{SU+I|1B{8Pvy&a`wUdzzls_7XDs}#DEIr@l>@I_4!XI$-=>`**{}YD=R8LYSPA~y zU5UW8y|wC}^G-*4v-Uz*+aA zq#E~>srTGXnYHe@S!Q?h%SZ0Un#bD$wD-t#=qD}9sF_9NTW+2K1IT&jd-8#LIU)DE6XXtUAI z8;t=81SOGC40icjw+SkNhKDUuN$NKKTKliEtS-lRIhXV_-=8{t@j(<{d_2b89 zS2k|m3O#|fj-6`;B>mJjt$vpE zY=o0*_Z4dxu4j7<1n*aaCAbYNcdNkStpQJk8!Q)inug)ZghXTICXg@+NCfoHXJC<+ zG)>}K#%hDr8`i>GSPX^3Ob{(kTk%*!8OU)4dQX9;8#+&c<1}=i0cSS^PAT-b!WHE4 z7YzReW9shViHtGj5G4B`F{TKI7yhndH?chdec}`Y?ajc@dI)XQp+!*^H3m~N8aw}l z7?YjRJ?VA~;mR%$ye(&-NbEW!0-gnz!DA*+sfllo0V2Kcn=3U5PbML`%Digf@}kAd zP&y+F{=#u7Dk(2kOp*pJm7ixWisZ{l%pZ+%l(_RsNvLzf(28{DSu9B=^rJF9&!UWg zb18f%mSSM&g-hNuFIifM)E4XvG-`D-mtyP?>LGR7iVw|;RubcCmAQ~qbyyL$s>jio zQpwd`L~8$-fk-XOF^am3NK<|7omP{<@U_+Cy)*p8M$;CnX}i^gT9DB{?V~+s9is1| zu)hUDw$DZP+0qneUAg%8wNJ$ZyDCK-Ci;QWR8?3!SOuOA|2Z7b{13-|KBbNZr75K{ z<`@ifCeA@3le{^lW^9pPQ8o3*kiw5-RV-yN42pror+TRx6e&~C500kf8rAu;UHnOC z6q@(eiPi&sLU&22xW3UM*0yaD6DV2U2no*fLPZmlUl!VHGOFFekIll-+bP&cO;DOr z>(im^PI=_2(K454`?d>z;?%&HK7?kCyp!LpFb|#+V)HeUzK) zAhA*;qoYjlQuOPp`h5ag4>KTULfneDA#q#c=0q0tnl=&k!2GI% zBT#mxw=4%E~$>dItKBzUiiWaKYTp5M&eXD}$mR~Y^ z52pvKoN)U4kyKdMPsj^j-ZLJzceHaw!Hu#oDDRDc7%Y=+_oTKQN-L>KrLwc*dIuv= zqpYr@nZ2E92m-RVr&<0;Yb{Uf>REuJquwevQgZ0|p_f$0INy*AfsiX2D_fkFqy)?7 zMT3act5(b)T6`6$(K*!qLCSHu#XZT`ohi+af&KCEI5H<&@e=)Pq0|&goi7jbE;Ug` z1;&a@9gmD6C$mW9?e8LyXbsN`HV)m*@en;ZvNt9c(IYfTaREI*`wI->+z%B&wHl(| zR^uS}+k*t?`JX~{TokY+4vv8OSjnP^6yRB8Fpnk{0JEfgaVtFnzf<^zi&@99%;=?TBTyUi(tP4P^oZi8=~GlX&4)|?}b|zi9{;Y$`i#H z8IFjo1VS?_8@+tBlbk>WqGA6nfeJ;=LTQ@ZSoJ5#o6G3_x&jp`1`G>2KJHdBesrl# z(uPhj`awDdiKBatVSyUfkdt4*OX9p=+2{>l)07=&6Xw-RK1j6SdIH)iF6Hzorno}N QFt*H=O0~!_da4lr0Gcc)Qvd(} literal 0 HcmV?d00001 diff --git a/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs new file mode 100644 index 000000000..c95f66fb2 --- /dev/null +++ b/datafusion/distributed/src/metrics/plan_dump/plan_dump_display.rs @@ -0,0 +1,872 @@ +use crate::metrics::generated::metrics::{self as pb, PlanDump, StageDump}; +use crate::metrics::metrics_proto::metric_proto_to_df; +use crate::metrics::plan_dump::derived_metrics::stage_derived_metrics; +use crate::metrics::plan_dump::plan_dump_generation::{ + HEAD_STAGE_ID, LEAF_DISPLAY, ZSTD_DICTIONARY, metric_task, metric_values, +}; +use crate::stage::{format_each_metric_by_task, metrics_by_task_id}; +use crate::{DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedMetricsFormat}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use datafusion::common::{ + HashMap, HashSet, Result, exec_datafusion_err, exec_err, internal_datafusion_err, +}; +use datafusion::physical_plan::metrics::{Metric, MetricsSet}; +use prost::Message; +use std::fmt::Write; +use std::io::Read; +use std::sync::{Arc, LazyLock}; +use zstd::dict::DecoderDictionary; + +/// [ZSTD_DICTIONARY] prepared once for decompressing, instead of on every dump. +static ZSTD_DECODER_DICTIONARY: LazyLock> = + LazyLock::new(|| DecoderDictionary::copy(ZSTD_DICTIONARY)); + +/// Options for displaying a plan dump. +#[derive(Debug, Clone)] +pub struct DisplayPlanDumpOptions { + /// The stage id to visualize. If none passed, the tree of stages is rendered. + stage_id: Option, + /// Whether to color the output with ANSI escape codes. + color: bool, + /// Whether metrics are displayed per task or aggregated across all tasks. + metrics_format: DistributedMetricsFormat, + /// If not empty, only the metrics whose name contains one of these are displayed. + metric_filters: Vec, + /// Whether metric names must be equal to a filter instead of containing it. + exact_metric_filters: bool, +} + +impl Default for DisplayPlanDumpOptions { + fn default() -> Self { + Self { + stage_id: None, + color: false, + metrics_format: DistributedMetricsFormat::PerTask, + metric_filters: vec![], + exact_metric_filters: false, + } + } +} + +impl DisplayPlanDumpOptions { + /// Renders the nodes of this stage with their metrics instead of the tree of stages. The head + /// of the plan, run by the coordinator, is stage 0. + pub fn with_stage_id(mut self, stage_id: usize) -> Self { + self.stage_id = Some(stage_id); + self + } + + /// Colors the output with ANSI escape codes, for displaying it in a terminal. + pub fn with_color(mut self, color: bool) -> Self { + self.color = color; + self + } + + /// Displays metrics per task, like `output_rows={0:10, 1:7}`, which is the default, or + /// aggregated across all tasks, like `output_rows=17`, which is shorter but hides how the work + /// was spread across tasks. + pub fn with_metrics_format(mut self, metrics_format: DistributedMetricsFormat) -> Self { + self.metrics_format = metrics_format; + self + } + + /// Displays only the metrics whose name contains `filter`, like `bytes` for `output_bytes` + /// and `spilled_bytes`. Adding several filters displays the metrics matching any of them. + pub fn with_metric_filter(mut self, filter: impl Into) -> Self { + self.metric_filters.push(filter.into()); + self + } + + /// Matches metric names that are equal to a filter, instead of the ones containing it, like + /// `cargo test -- --exact` does with test names: `output_bytes` then leaves out + /// `estimated_output_bytes`. + pub fn with_exact_metric_filters(mut self, exact: bool) -> Self { + self.exact_metric_filters = exact; + self + } + + fn matches_metric_filters(&self, name: &str) -> bool { + let mut filters = self.metric_filters.iter(); + match self.exact_metric_filters { + true => filters.any(|filter| name == filter), + false => filters.any(|filter| name.contains(filter.as_str())), + } + } +} + +/// Displays a plan dump generated by [generate_plan_dump](crate::generate_plan_dump). See +/// [DisplayPlanDumpOptions] for different types of rendering. +/// +/// This is the function used by DFD's CLI for displaying plan dumps in the terminal. +pub fn display_plan_dump(plan: String, opts: DisplayPlanDumpOptions) -> Result { + let compressed = STANDARD + .decode(plan.trim()) + .map_err(|err| exec_datafusion_err!("Plan dump is not valid base64: {err}"))?; + let mut encoded = vec![]; + zstd::stream::Decoder::with_prepared_dictionary( + compressed.as_slice(), + &ZSTD_DECODER_DICTIONARY, + ) + .and_then(|mut decoder| decoder.read_to_end(&mut encoded)) + .map_err(|err| exec_datafusion_err!("Plan dump is not zstd-compressed: {err}"))?; + let head = StageDump::decode(encoded.as_slice()) + .map_err(|err| exec_datafusion_err!("Plan dump does not contain a plan: {err}"))?; + + let mut head = decode_stage(head, None)?; + // Metrics are filtered once decoded, so that the metrics derived from them do not depend on + // the filters. + if !opts.metric_filters.is_empty() { + filter_stage_metrics(&mut head, &|name| opts.matches_metric_filters(name)); + } + + let paint = Paint(opts.color); + let format = opts.metrics_format; + let single_node = input_stages(&head).is_empty(); + let tree = match (opts.stage_id, &head.plan) { + // A plan without stages is a single-node plan, which is displayed as its nodes. + (None | Some(HEAD_STAGE_ID), Some(plan)) if single_node => { + operator_tree(plan, format, paint) + } + (None, _) => stage_tree(&head, format, paint), + (Some(id), _) => { + let Some(stage) = find_stage(&head, id as u64) else { + return exec_err!("Stage {id} not found in the plan dump"); + }; + let mut node = stage_node(stage, format, paint); + if let Some(plan) = &stage.plan { + node.children.push(operator_tree(plan, format, paint)); + } + node + } + }; + let mut out = String::new(); + tree.render("", "", paint, &mut out) + .map_err(|err| internal_datafusion_err!("Failed to render plan dump: {err}"))?; + Ok(out) +} + +/// A stage of a plan dump, with its metrics decoded for displaying them. +pub(super) struct DecodedStage { + pub(super) id: u64, + pub(super) tasks: u64, + pub(super) partitions: u64, + /// The kind of network boundary reading the stage, like `Shuffle`, which the head stage does + /// not have. + pub(super) boundary_kind: Option, + /// The stage's own metrics, and the ones derived for it. + pub(super) metrics: MetricsSet, + pub(super) plan: Option, +} + +/// A node of a plan dump, with its metrics decoded for displaying them. +pub(super) struct DecodedPlan { + pub(super) display: String, + pub(super) metrics: MetricsSet, + pub(super) children: Vec, + /// For network boundaries, the stage they read from. Its plan is not part of `children`. + pub(super) input_stage: Option>, +} + +/// Decodes a stage read by `boundary`, a network boundary with its display and metrics, and adds +/// the metrics derived for it. +fn decode_stage(stage: StageDump, boundary: Option<(&str, &MetricsSet)>) -> Result { + let mut decoded = DecodedStage { + id: stage.stage_id, + tasks: stage.tasks, + partitions: stage.partitions, + boundary_kind: boundary.map(|(display, _)| boundary_kind(display).to_string()), + metrics: undump_metrics(&stage.metrics)?, + plan: stage.plan_dump.map(|plan| decode_plan(*plan)).transpose()?, + }; + for metric in stage_derived_metrics(&decoded, boundary.map(|(_, metrics)| metrics)) { + decoded.metrics.push(metric); + } + Ok(decoded) +} + +fn decode_plan(plan: PlanDump) -> Result { + let metrics = undump_metrics(&plan.metrics)?; + let input_stage = match plan.input_stage { + Some(stage) => Some(Box::new(decode_stage( + *stage, + Some((&plan.display, &metrics)), + )?)), + None => None, + }; + Ok(DecodedPlan { + children: plan + .children + .into_iter() + .map(decode_plan) + .collect::>()?, + display: plan.display, + metrics, + input_stage, + }) +} + +/// Leaves only the metrics whose name `matches`, in `stage`, in its nodes and in the stages it +/// reads from. +fn filter_stage_metrics(stage: &mut DecodedStage, matches: &dyn Fn(&str) -> bool) { + stage.metrics = matching_metrics(&stage.metrics, matches); + if let Some(plan) = &mut stage.plan { + filter_plan_metrics(plan, matches); + } +} + +fn filter_plan_metrics(plan: &mut DecodedPlan, matches: &dyn Fn(&str) -> bool) { + plan.metrics = matching_metrics(&plan.metrics, matches); + for child in &mut plan.children { + filter_plan_metrics(child, matches); + } + if let Some(stage) = &mut plan.input_stage { + filter_stage_metrics(stage, matches); + } +} + +fn matching_metrics(metrics: &MetricsSet, matches: &dyn Fn(&str) -> bool) -> MetricsSet { + let mut matching = MetricsSet::new(); + for metric in metrics.iter() { + if matches(metric.value().name()) { + matching.push(Arc::clone(metric)); + } + } + matching +} + +/// Reverses how a dump stores metrics, as one column per metric, giving each metric of a column +/// its task label back. +fn undump_metrics(columns: &[pb::MetricColumn]) -> Result { + let mut metrics = MetricsSet::new(); + for column in columns { + let Some(template) = &column.metric else { + continue; + }; + let width = metric_values(&mut template.clone())?.len(); + if column.values.len() != column.tasks.len() * width { + return exec_err!("Plan dump has a metric column with a wrong number of values"); + } + for (task, values) in column.tasks.iter().zip(column.values.chunks_exact(width)) { + let mut metric = template.clone(); + for (slot, value) in metric_values(&mut metric)?.into_iter().zip(values) { + *slot = *value; + } + metric.labels.push(pb::Label { + name: DISTRIBUTED_DATAFUSION_TASK_ID_LABEL.to_string(), + value: task.to_string(), + }); + metrics.push(metric_proto_to_df(metric)?); + } + } + Ok(metrics) +} + +/// A node of a rendered plan dump: a label followed by one line per metric, and its children. +struct Node { + label: String, + metrics: Vec, + children: Vec, +} + +impl Node { + fn render( + &self, + prefix: &str, + child_prefix: &str, + paint: Paint, + out: &mut String, + ) -> std::fmt::Result { + writeln!(out, "{}{}", paint.dim(prefix), self.label)?; + let guide = match self.children.is_empty() { + true => " ", + false => "│ ", + }; + let metric_prefix = paint.dim(&format!("{child_prefix}{guide}")); + for metric in &self.metrics { + writeln!(out, "{metric_prefix}{metric}")?; + } + for (i, child) in self.children.iter().enumerate() { + let (branch, continuation) = match i == self.children.len() - 1 { + true => ("└─ ", " "), + false => ("├─ ", "│ "), + }; + child.render( + &format!("{child_prefix}{branch}"), + &format!("{child_prefix}{continuation}"), + paint, + out, + )?; + } + Ok(()) + } +} + +/// A stage with its metrics. +fn stage_node(stage: &DecodedStage, format: DistributedMetricsFormat, paint: Paint) -> Node { + let label = match (stage.id, &stage.boundary_kind) { + (0, None) => paint.stage("DistributedExec"), + (id, boundary_kind) => { + let boundary_kind = (boundary_kind.as_ref()) + .map(|kind| format!(" {}", paint.boundary(&format!("({kind})")))) + .unwrap_or_default(); + let tasks = paint.metric(&format!("tasks={}", stage.tasks)); + let partitions = paint.metric(&format!("partitions={}", stage.partitions)); + format!( + "{}{boundary_kind} {tasks} {partitions}", + paint.stage(&format!("Stage {id}")) + ) + } + }; + Node { + label, + metrics: metric_lines(&stage.metrics, format, paint), + children: vec![], + } +} + +/// Every stage as a tree, each stage under the stage reading from it. +fn stage_tree(stage: &DecodedStage, format: DistributedMetricsFormat, paint: Paint) -> Node { + let mut node = stage_node(stage, format, paint); + node.children = (input_stages(stage).into_iter()) + .map(|input| stage_tree(input, format, paint)) + .collect(); + node +} + +/// The nodes of a stage as a tree. Network boundaries are leaves: the stages they read from are +/// rendered on their own. +fn operator_tree(plan: &DecodedPlan, format: DistributedMetricsFormat, paint: Paint) -> Node { + if plan.display == LEAF_DISPLAY { + return leaf_tree(plan, format, paint); + } + Node { + label: paint.operator(&plan.display), + metrics: metric_lines(&plan.metrics, format, paint), + children: (plan.children.iter()) + .map(|child| operator_tree(child, format, paint)) + .collect(), + } +} + +/// Like [crate::display_plan_ascii], each `t: …` variant of a +/// [DistributedLeafExec](crate::DistributedLeafExec) shows the metrics of the task that ran it. +/// Aggregated across tasks, they are shown on the leaf instead. +fn leaf_tree(plan: &DecodedPlan, format: DistributedMetricsFormat, paint: Paint) -> Node { + let (leaf_metrics, by_task) = match format { + DistributedMetricsFormat::PerTask => (vec![], metrics_by_task_id(&plan.metrics)), + DistributedMetricsFormat::Aggregated => { + (metric_lines(&plan.metrics, format, paint), HashMap::new()) + } + }; + let variant = |task: usize, variant: &DecodedPlan| Node { + label: paint.operator(&variant.display), + metrics: by_task + .get(&task) + .map(|metrics| metric_lines(metrics, format, paint)) + .unwrap_or_default(), + children: vec![], + }; + Node { + label: paint.operator(&plan.display), + metrics: leaf_metrics, + children: (plan.children.iter().enumerate()) + .map(|(task, child)| variant(task, child)) + .collect(), + } +} + +fn find_stage(stage: &DecodedStage, id: u64) -> Option<&DecodedStage> { + if stage.id == id { + return Some(stage); + } + (input_stages(stage).into_iter()).find_map(|input| find_stage(input, id)) +} + +/// The stages `stage` reads from, in plan order. +fn input_stages(stage: &DecodedStage) -> Vec<&DecodedStage> { + fn collect<'a>(plan: &'a DecodedPlan, inputs: &mut Vec<&'a DecodedStage>) { + inputs.extend(plan.input_stage.as_deref()); + for child in &plan.children { + collect(child, inputs); + } + } + let mut inputs = vec![]; + if let Some(plan) = &stage.plan { + collect(plan, &mut inputs); + } + inputs +} + +/// `Shuffle` for a `[Stage 2] => NetworkShuffleExec: …` node. +fn boundary_kind(display: &str) -> &str { + let name = display.split_once("=> ").map_or(display, |(_, rest)| rest); + let name = name.split(':').next().unwrap_or(name); + name.trim_start_matches("Network").trim_end_matches("Exec") +} + +/// One `name=value` line per metric, with per-task values as `name={task:value, …}`. A metric only +/// reported by task 0, like the ones of a plan that is not distributed, displays as `name=value`, +/// and so do all metrics when they are aggregated across tasks. +fn metric_lines( + metrics: &MetricsSet, + format: DistributedMetricsFormat, + paint: Paint, +) -> Vec { + // Metrics without a task label are added up across tasks when formatted. + let per_task: HashSet<&str> = match format { + DistributedMetricsFormat::PerTask => (metrics.iter()) + .filter(|metric| metric_task(metric).is_ok_and(|task| task != 0)) + .map(|metric| metric.value().name()) + .collect(), + DistributedMetricsFormat::Aggregated => HashSet::new(), + }; + let mut displayed = MetricsSet::new(); + for metric in metrics.iter() { + displayed.push(match per_task.contains(metric.value().name()) { + true => Arc::clone(metric), + false => without_task_label(metric), + }); + } + format_each_metric_by_task(&displayed) + .iter() + .map(|metric| paint.metric(metric)) + .collect() +} + +fn without_task_label(metric: &Metric) -> Arc { + let labels = (metric.labels().iter()) + .filter(|label| label.name() != DISTRIBUTED_DATAFUSION_TASK_ID_LABEL) + .cloned() + .collect(); + Arc::new(Metric::new_with_labels( + metric.value().clone(), + None, + labels, + )) +} + +/// Where the parameter name that `before` ends with starts, like `mode` in `…, mode`, if it is a +/// whole lowercase word. +fn param_name_start(before: &str) -> Option { + let start = (before.char_indices().rev()) + .take_while(|(_, c)| c.is_ascii_lowercase() || *c == '_') + .last()? + .0; + let at_word_start = + (before[..start].chars().next_back()).is_none_or(|c| !(c.is_alphanumeric() || c == '_')); + at_word_start.then_some(start) +} + +/// Colors the parts of a rendered plan dump with ANSI escape codes when enabled, similar to how +/// code is highlighted. When disabled, text is returned as it is. +#[derive(Debug, Clone, Copy)] +struct Paint(bool); + +impl Paint { + fn style(self, code: &str, text: &str) -> String { + match self.0 && !text.is_empty() { + true => format!("\x1b[{code}m{text}\x1b[0m"), + false => text.to_string(), + } + } + + /// Tree guides, task indexes and separators. + fn dim(self, text: &str) -> String { + self.style("90", text) + } + + fn stage(self, text: &str) -> String { + self.style("1;36", text) + } + + fn boundary(self, text: &str) -> String { + self.style("35", text) + } + + fn value(self, text: &str) -> String { + self.style("33", text) + } + + /// A `name=value` metric, with per-task values as `name={task:value, …}`. + fn metric(self, metric: &str) -> String { + let Some((name, value)) = metric.split_once('=') else { + return metric.to_string(); + }; + let value = match value.strip_prefix('{').and_then(|v| v.strip_suffix('}')) { + Some(tasks) => self.tasks(tasks), + None => self.value(value), + }; + format!("{name}{}{value}", self.dim("=")) + } + + /// The `task:value, …` values of a metric for each task, within braces. + fn tasks(self, tasks: &str) -> String { + let tasks = (tasks.split(", ")) + .map(|task| match task.split_once(':') { + Some((index, value)) => { + format!("{}{}", self.dim(&format!("{index}:")), self.value(value)) + } + None => self.value(task), + }) + .collect::>() + .join(&self.dim(", ")); + format!("{}{tasks}{}", self.dim("{"), self.dim("}")) + } + + /// An operator, like `[Stage 1] => NetworkShuffleExec: input_tasks=3`, with the stage it reads + /// from, its name and its parameter names highlighted. + fn operator(self, label: &str) -> String { + let mut out = String::new(); + let mut rest = label; + // The `t: ` prefix of the variants of a DistributedLeafExec. + if let Some((task, after)) = rest.split_once(": ") + && task.len() > 1 + && task.starts_with('t') + && task[1..].chars().all(|c| c.is_ascii_digit()) + { + out += &self.dim(&format!("{task}: ")); + rest = after; + } + // The `[Stage N] => ` prefix of network boundaries. + if rest.starts_with("[Stage ") + && let Some((input, after)) = rest.split_once(" => ") + { + out += &self.stage(input); + out += &self.dim(" => "); + rest = after; + } + let name_len = rest + .find(|c: char| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(rest.len()); + out += &self.style("1;32", &rest[..name_len]); + out += &self.params(&rest[name_len..]); + out + } + + /// Highlights parameter names, like `mode` in `mode=Partitioned`. + fn params(self, text: &str) -> String { + let mut out = String::new(); + let mut rest = text; + while let Some(eq) = rest.find('=') { + let before = &rest[..eq]; + match param_name_start(before) { + Some(start) => { + out += &before[..start]; + out += &self.style("34", &before[start..]); + out += &self.dim("="); + } + None => { + out += before; + out += "="; + } + } + rest = &rest[eq + 1..]; + } + out + rest + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::assert_snapshot; + use crate::generate_plan_dump; + use crate::test_utils::plans::TestPlanBuilder; + use insta::with_settings; + + #[tokio::test] + async fn single_node_plan() -> Result<()> { + let query = r#"SELECT "RainToday", count(*) FROM weather GROUP BY "RainToday""#; + let plan = TestPlanBuilder::default() + .distributed_planner(false) + .physical_plan(query) + .await; + + let dump = generate_plan_dump(&plan)?; + + let display = display_plan_dump(dump, DisplayPlanDumpOptions::default())?; + assert_snapshot!(display, @r" + ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + └─ RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=3 + └─ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + └─ DataSourceExec: file_groups={3 groups}, projection=[RainToday], file_type=parquet + "); + Ok(()) + } + + #[tokio::test] + async fn distributed_plan_stages() -> Result<()> { + let query = r#"SELECT "RainToday", count(*) FROM weather GROUP BY "RainToday""#; + let plan = TestPlanBuilder::default().physical_plan(query).await; + + let dump = generate_plan_dump(&plan)?; + + let display = display_plan_dump(dump, DisplayPlanDumpOptions::default())?; + assert_snapshot!(display, @r" + DistributedExec + └─ Stage 2 (Coalesce) tasks=3 partitions=4 + └─ Stage 1 (Shuffle) tasks=3 partitions=12 + "); + Ok(()) + } + + #[tokio::test] + async fn distributed_plan_stage() -> Result<()> { + let query = r#"SELECT "RainToday", count(*) FROM weather GROUP BY "RainToday""#; + let plan = TestPlanBuilder::default().physical_plan(query).await; + + let dump = generate_plan_dump(&plan)?; + + let opts = DisplayPlanDumpOptions::default().with_stage_id(2); + let display = display_plan_dump(dump, opts)?; + assert_snapshot!(display, @r" + Stage 2 (Coalesce) tasks=3 partitions=4 + └─ ProjectionExec: expr=[RainToday@0 as RainToday, count(Int64(1))@1 as count(*)] + └─ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + └─ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + "); + Ok(()) + } + + #[tokio::test] + async fn distributed_leaf() -> Result<()> { + let query = r#"SELECT "MinTemp", "RainToday" FROM weather WHERE "Rainfall" > 0"#; + let plan = TestPlanBuilder::default().physical_plan(query).await; + + let dump = generate_plan_dump(&plan)?; + + let opts = DisplayPlanDumpOptions::default().with_stage_id(1); + let display = display_plan_dump(dump, opts)?; + assert_snapshot!(display, @r" + Stage 1 (Coalesce) tasks=3 partitions=12 + └─ FilterExec: Rainfall@1 > 0, projection=[MinTemp@0, RainToday@2] + └─ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + └─ DistributedLeafExec: + ├─ t0: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + ├─ t1: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + └─ t2: DataSourceExec: file_groups={3 groups}, projection=[MinTemp, Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 0, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 0, required_guarantees=[] + "); + Ok(()) + } + + #[tokio::test] + async fn stages_show_the_cpu_time_of_their_nodes() -> Result<()> { + let query = r#"SELECT "MinTemp", "RainToday" FROM weather WHERE "Rainfall" > 0"#; + let plan = TestPlanBuilder::default() + .executed_physical_plan(query) + .await; + + let dump = generate_plan_dump(&plan)?; + + let display = display_plan_dump(dump, DisplayPlanDumpOptions::default())?; + let filters = vec![(TIMES, "