Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ datafusion-proto-common = { git = "https://github.com/apache/datafusion", tag =
datafusion-substrait = { git = "https://github.com/apache/datafusion", tag = "56.0.0-rc1" }

ctor = { version = "1.0" }
datasketches = { version = "0.6", default-features = false, features = ["hll"] }
insta = "1.47"
itertools = "0.15"
mimalloc = { version = "0.1" }
Expand Down
1 change: 1 addition & 0 deletions ballista/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ clap = { workspace = true, optional = true }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
datafusion-proto-common = { workspace = true }
datasketches = { workspace = true }
futures = { workspace = true }
log = { workspace = true }
lru = { version = "0.19", optional = true }
Expand Down
10 changes: 2 additions & 8 deletions ballista/core/proto/ballista.proto
Original file line number Diff line number Diff line change
Expand Up @@ -780,14 +780,8 @@ message PartitionStats {
int64 num_rows = 1;
int64 num_batches = 2;
int64 num_bytes = 3;
repeated ColumnStats column_stats = 4;
}

message ColumnStats {
datafusion_common.ScalarValue min_value = 1;
datafusion_common.ScalarValue max_value = 2;
uint32 null_count = 3;
uint32 distinct_count = 4;
reserved 4;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reserving field 4 is the right call on the wire side. Since ColumnStats and PartitionStats::column_stats were public in 55.0.0, downstream code that builds a PartitionStats with a struct literal will stop compiling. Could you add a short note to docs/source/upgrading/56.0.0.md? We should also add the api change label.

reserved "column_stats";
}

message OperatorMetricsSet {
Expand Down
10 changes: 5 additions & 5 deletions ballista/core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -612,8 +612,8 @@ static CONFIG_ENTRIES: LazyLock<HashMap<String, ConfigEntry>> = LazyLock::new(||
column with HyperLogLog while writing shuffle output, so the adaptive \
planner can estimate join cardinalities downstream of the shuffle. \
Costs hashing every key value on the write path and one sketch per \
key column in each task's status update (4 KiB with the default \
precision).".to_string(),
key column in each task's status update (at most about 2 KiB with \
the default precision).".to_string(),
DataType::Boolean,
Some(true.to_string()),
),
Expand All @@ -622,9 +622,9 @@ static CONFIG_ENTRIES: LazyLock<HashMap<String, ConfigEntry>> = LazyLock::new(||
"Precision of the HyperLogLog sketches that \
`ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 \
(values outside that range are clamped). A sketch has \
2^precision one-byte registers and a standard error of about \
1.04 / 2^(precision / 2): 10 is 1 KiB and 3.3%, 12 is 4 KiB and \
1.6%, 14 is 16 KiB and 0.8%. The scheduler keeps one sketch per \
2^precision registers, sent at 4 bits each, and a standard error of \
about 1.04 / 2^(precision / 2): 10 is 512 B and 3.3%, 12 is 2 KiB \
and 1.6%, 14 is 8 KiB and 0.8%. The scheduler keeps one sketch per \
key column for every task of a stage, so lower it for stages with \
many thousands of tasks.".to_string(),
DataType::UInt16,
Expand Down
2 changes: 0 additions & 2 deletions ballista/core/src/execution_plans/shuffle_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,6 @@ pub fn stats_for_partition(
num_fields: usize,
partition_locations: &[Vec<PartitionLocation>],
) -> Result<Statistics> {
// TODO stats: add column statistics to PartitionStats
let (num_rows, total_byte_size) = partition_locations
.iter()
.map(|location| {
Expand Down Expand Up @@ -654,7 +653,6 @@ pub fn stats_for_partitions(
num_fields: usize,
partition_stats: impl Iterator<Item = PartitionStats>,
) -> Statistics {
// TODO stats: add column statistics to PartitionStats
let (num_rows, total_byte_size) =
partition_stats.fold((Some(0), Some(0)), |(num_rows, total_byte_size), part| {
// if any statistic is unkown it makes the entire statistic unkown
Expand Down
26 changes: 11 additions & 15 deletions ballista/core/src/execution_plans/shuffle_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ use crate::execution_plans::{
use crate::extension::SessionConfigExt;
use crate::utils;

use crate::hll::HyperLogLog;
use crate::hll::{HyperLogLog, HyperLogLogUnion};
use crate::serde::protobuf::{
ColumnSketch, ShuffleWritePartition, SketchType, TaskColumnStats,
};
Expand Down Expand Up @@ -259,7 +259,7 @@ pub(crate) struct ColumnStatsCollector(Arc<Mutex<ColumnStatsTotals>>);
struct ColumnStatsTotals {
null_counts: Vec<u64>,
/// `(column index, sketch)` for each sketched column.
sketches: Vec<(usize, HyperLogLog)>,
sketches: Vec<(usize, HyperLogLogUnion)>,
}

/// The columns a writer sketches: the plain column references among the hash
Expand Down Expand Up @@ -291,7 +291,7 @@ impl ColumnStatsCollector {
null_counts: vec![0; num_columns],
sketches: sketched
.iter()
.map(|&column| (column, HyperLogLog::new(precision)))
.map(|&column| (column, HyperLogLogUnion::new(precision)))
.collect(),
};
}
Expand Down Expand Up @@ -360,7 +360,7 @@ impl ColumnStatsCollector {
if let Some(entry) = stats.get_mut(*column) {
entry.sketches.push(ColumnSketch {
r#type: SketchType::Hll as i32,
data: sketch.as_bytes().to_vec(),
data: sketch.to_sketch().to_bytes(),
});
}
}
Expand Down Expand Up @@ -420,9 +420,7 @@ impl Drop for ColumnStatsStream {
}
for ((_, slot), (_, sketch)) in total.sketches.iter_mut().zip(&self.sketches)
{
// Both were made at the precision `reset` chose, so they merge.
let merged = slot.merge(sketch);
debug_assert!(merged, "stream sketch and total differ in precision");
slot.update(sketch);
}
}
}
Expand Down Expand Up @@ -1680,10 +1678,9 @@ mod tests {
let [sketch] = stats[0].sketches.as_slice() else {
panic!("expected one sketch on the hash key, got {:?}", stats[0]);
};
assert_eq!(sketch.data.len(), 1 << crate::hll::DEFAULT_PRECISION);
let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data)
.unwrap()
.count();
let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap();
assert_eq!(hll.precision(), crate::hll::DEFAULT_PRECISION);
let distinct = hll.count();
assert!((95..=105).contains(&distinct), "estimated {distinct}");
assert!(stats[1].sketches.is_empty());
Ok(())
Expand All @@ -1704,10 +1701,9 @@ mod tests {
let [sketch] = stats[0].sketches.as_slice() else {
panic!("expected one sketch on the hash key, got {:?}", stats[0]);
};
assert_eq!(sketch.data.len(), 1 << 10);
let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data)
.unwrap()
.count();
let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap();
assert_eq!(hll.precision(), 10);
let distinct = hll.count();
// Standard error is 3.3% here.
assert!((80..=120).contains(&distinct), "estimated {distinct}");
Ok(())
Expand Down
5 changes: 3 additions & 2 deletions ballista/core/src/execution_plans/sort_shuffle/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2331,8 +2331,9 @@ mod tests {
let [sketch] = stats[0].sketches.as_slice() else {
panic!("expected one sketch on the hash key, got {:?}", stats[0]);
};
assert_eq!(sketch.data.len(), 1 << 10);
let distinct = HyperLogLog::from_bytes(&sketch.data).unwrap().count();
let hll = HyperLogLog::from_bytes(&sketch.data).unwrap();
assert_eq!(hll.precision(), 10);
let distinct = hll.count();
// 1500 distinct values at a standard error of 3.3%.
assert!((1200..=1800).contains(&distinct), "estimated {distinct}");
Ok(())
Expand Down
Loading
Loading