Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions datafusion/distributed/src/coordinator/prepare_dynamic_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ pub(super) async fn prepare_dynamic_plan(

let head_stage = inject_network_boundaries(
Arc::clone(base_plan),
// TODO: Inject here a plan remainder replanner for reshaping the topology of the plan based on the runtime
// collected statistics.
|plan, _nb_ctx| Ok(plan),
DynamicPlanBuilder {
query_coordinator,
plans_for_viz: &plans_for_viz,
Expand Down Expand Up @@ -108,8 +111,9 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> {
// Propagate the final task_count inferred based on runtime statistics and compute cost.
// Here is where leaf nodes are scaled up by ScaleUpLeafNodeHandler, and the
// plan is finally left ready for distribution.
input_stage.plan =
nb_ctx.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?;
input_stage.plan = nb_ctx
.propagate_task_count_until_network_boundaries(&input_stage.plan, task_count)?
.data;
input_stage.tasks = task_count.as_usize();
// In order to infer the compute the cost of the stage above this one, here a sampler
// is injected to gather runtime statistics.
Expand Down Expand Up @@ -146,8 +150,12 @@ impl NetworkBoundaryBuilder for DynamicPlanBuilder<'_> {
let (stats, consumer_tc) = if nb_type == TypeId::of::<NetworkCoalesceExec>() {
(None, TaskCountAnnotation::exact(NonZeroUsize::MIN, 0.0))
} else {
let (stats, new_metrics) =
gather_runtime_statistics(load_info_rxs, &input_stage.plan).await?;
let (stats, new_metrics) = gather_runtime_statistics(
load_info_rxs,
&input_stage.plan,
nb_ctx.d_cfg.dynamic_bytes_ready_sample_percentage,
)
.await?;
metrics.extend(new_metrics);
// returning Desired(0) here is our way to tell the planner that we don't care
// about the task count assigned to the network boundary in the consumer stage,
Expand Down Expand Up @@ -233,11 +241,10 @@ impl PlanReconstructor {
async fn gather_runtime_statistics(
per_task_load_info_stream: Vec<impl Stream<Item = LoadInfo> + Unpin>,
plan: &Arc<dyn ExecutionPlan>,
// Fraction of partitions with non-zero sampled bytes that need to report before considering
// that there's enough samples collected to make a final decision.
bytes_ready_sample_percentage: f64,
) -> Result<(Statistics, MetricsSet)> {
/// How many LoadInfo samples with non-zero sampled bytes should be gathered before considering
/// that there's enough samples collected to make a final decision. The higher this value is,
/// the more execution is delayed because of sample collection, but better sampling precision.
const BYTES_READY_SAMPLE_PERCENTAGE: f32 = 0.2;
/// If the leaf nodes in the provided `plan` do not provide any row count estimation, there's
/// no way for us here to estimate the % of completion of the stage during sampling. This is
/// the default % of completion for those cases.
Expand All @@ -249,8 +256,8 @@ async fn gather_runtime_statistics(
};
let n_cols = sampler.schema().fields.len();

fn apply_pct(value: usize, pct: f32) -> usize {
(value as f32 * pct).round() as usize
fn apply_pct(value: usize, pct: f64) -> usize {
(value as f64 * pct).round() as usize
}

let partitions_per_task = sampler.partition_samplers.len();
Expand Down Expand Up @@ -283,7 +290,7 @@ async fn gather_runtime_statistics(

// Short circuit if we collected enough bytes_ready measurements.
if partitions_with_bytes_ready_done
>= apply_pct(total_partitions, BYTES_READY_SAMPLE_PERCENTAGE).max(1)
>= apply_pct(total_partitions, bytes_ready_sample_percentage).max(1)
{
break;
}
Expand Down
43 changes: 43 additions & 0 deletions datafusion/distributed/src/distributed_ext.rs
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,21 @@ pub trait DistributedExt: Sized {
dynamic_bytes_per_partition: usize,
) -> Result<(), DataFusionError>;

/// Fraction of a stage's partitions that need to report sampled bytes before the dynamic
/// task count allocator estimates the stage's runtime statistics. Higher values delay
/// execution longer while sampling, but produce more precise estimations.
fn with_distributed_dynamic_bytes_ready_sample_percentage(
self,
dynamic_bytes_ready_sample_percentage: f64,
) -> Result<Self, DataFusionError>;

/// Same as [DistributedExt::with_distributed_dynamic_bytes_ready_sample_percentage] but with
/// an in-place mutation.
fn set_distributed_dynamic_bytes_ready_sample_percentage(
&mut self,
dynamic_bytes_ready_sample_percentage: f64,
) -> Result<(), DataFusionError>;

/// Target throughput in bytes per partition per second used by the dynamic task count
/// allocator to decide how many tasks to assign to each stage based on runtime statistics.
fn with_distributed_local_worker_context(
Expand Down Expand Up @@ -1059,6 +1074,15 @@ impl DistributedExt for SessionConfig {
Ok(())
}

fn set_distributed_dynamic_bytes_ready_sample_percentage(
&mut self,
dynamic_bytes_ready_sample_percentage: f64,
) -> Result<(), DataFusionError> {
let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?;
d_cfg.dynamic_bytes_ready_sample_percentage = dynamic_bytes_ready_sample_percentage;
Ok(())
}

fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext) {
self.set_extension(Arc::new(local_worker_context));
}
Expand Down Expand Up @@ -1191,6 +1215,10 @@ impl DistributedExt for SessionConfig {
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result<Self, DataFusionError>;

#[call(set_distributed_dynamic_bytes_ready_sample_percentage)]
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<Self, DataFusionError>;

#[call(set_distributed_local_worker_context)]
#[expr($;self)]
fn with_distributed_local_worker_context(mut self, local_worker_context: LocalWorkerContext) -> Self;
Expand Down Expand Up @@ -1362,6 +1390,11 @@ impl DistributedExt for SessionStateBuilder {
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result<Self, DataFusionError>;

fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>;
#[call(set_distributed_dynamic_bytes_ready_sample_percentage)]
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<Self, DataFusionError>;

fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext);
#[call(set_distributed_local_worker_context)]
#[expr($;self)]
Expand Down Expand Up @@ -1542,6 +1575,11 @@ impl DistributedExt for SessionState {
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_per_partition(mut self, dynamic_bytes_per_partition: usize) -> Result<Self, DataFusionError>;

fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>;
#[call(set_distributed_dynamic_bytes_ready_sample_percentage)]
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_ready_sample_percentage(mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<Self, DataFusionError>;

fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext);
#[call(set_distributed_local_worker_context)]
#[expr($;self)]
Expand Down Expand Up @@ -1715,6 +1753,11 @@ impl DistributedExt for SessionContext {
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_per_partition(self, dynamic_bytes_per_partition: usize) -> Result<Self, DataFusionError>;

fn set_distributed_dynamic_bytes_ready_sample_percentage(&mut self, dynamic_bytes_ready_sample_percentage: f64) -> Result<(), DataFusionError>;
#[call(set_distributed_dynamic_bytes_ready_sample_percentage)]
#[expr($?;Ok(self))]
fn with_distributed_dynamic_bytes_ready_sample_percentage(self, dynamic_bytes_ready_sample_percentage: f64) -> Result<Self, DataFusionError>;

fn set_distributed_local_worker_context(&mut self, local_worker_context: LocalWorkerContext);
#[call(set_distributed_local_worker_context)]
#[expr($;self)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ extensions_options! {
/// If `dynamic_task_count` is enabled, this value is the amount of bytes each
/// partition is expected to handle. Lower values will result in greater parallelism.
pub dynamic_bytes_per_partition: usize, default = 16 * 1024 * 1024
/// If `dynamic_task_count` is enabled, this is the fraction of a stage's partitions that
/// need to report sampled bytes before the stage's runtime statistics are estimated. Higher
/// values delay execution longer while sampling, but produce more precise estimations. With
/// `1.0`, every partition is sampled before estimating.
pub dynamic_bytes_ready_sample_percentage: f64, default = 0.2
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,13 @@ pub fn create_distributed_plan(
}

// Compute per-node task counts and inject `Network*Exec` nodes at the stage boundaries.
plan = inject_network_boundaries(plan, CardinalityBasedNetworkBoundaryBuilder, session_cfg)
.await?;
plan = inject_network_boundaries(
plan,
|plan, _nb_ctx| Ok(plan),
CardinalityBasedNetworkBoundaryBuilder,
session_cfg,
)
.await?;

plan = prepare_network_boundaries(plan)?;
if !plan.exists(|plan| Ok(plan.is_network_boundary()))? {
Expand Down
Loading
Loading