From 34d9bf4a5c10c52887b05952f8e1bd39df880e51 Mon Sep 17 00:00:00 2001 From: Bhargava Vadlamani Date: Sat, 14 Nov 2026 15:37:02 -0500 Subject: [PATCH 1/2] setup datasketches and update hll impl --- Cargo.lock | 13 +- Cargo.toml | 1 + ballista/core/Cargo.toml | 1 + ballista/core/proto/ballista.proto | 10 +- ballista/core/src/config.rs | 10 +- .../src/execution_plans/shuffle_reader.rs | 2 - .../src/execution_plans/shuffle_writer.rs | 26 +- .../execution_plans/sort_shuffle/writer.rs | 5 +- ballista/core/src/hll.rs | 266 ++++++++---------- ballista/core/src/serde/generated/ballista.rs | 16 +- ballista/core/src/serde/scheduler/to_proto.rs | 1 - .../scheduler/src/state/execution_stage.rs | 48 ++-- 12 files changed, 170 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e171274a5..c349a82a00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1130,6 +1130,7 @@ dependencies = [ "datafusion", "datafusion-proto", "datafusion-proto-common", + "datasketches", "futures", "log", "lru 0.19.0", @@ -3061,6 +3062,12 @@ dependencies = [ "url", ] +[[package]] +name = "datasketches" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17518a0e88b8a18c546a855d9db6455caaea7865774364bff759edf6a4731aca" + [[package]] name = "defmt" version = "1.1.1" @@ -5532,7 +5539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5553,7 +5560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7003,7 +7010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 7283528d1f..5b5619bdee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ datafusion-proto-common = "55" datafusion-substrait = "55" ctor = { version = "1.0" } +datasketches = { version = "0.6", default-features = false, features = ["hll"] } insta = "1.47" itertools = "0.15" mimalloc = { version = "0.1" } diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 28704b122c..be9f0fc085 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -51,6 +51,7 @@ clap = { workspace = true, optional = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-proto-common = { workspace = true } +datasketches = { workspace = true } futures = { workspace = true } log = { workspace = true } lru = { version = "0.19", optional = true } diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto index 146f8c9369..0bde17f1b3 100644 --- a/ballista/core/proto/ballista.proto +++ b/ballista/core/proto/ballista.proto @@ -738,14 +738,8 @@ message PartitionStats { int64 num_rows = 1; int64 num_batches = 2; int64 num_bytes = 3; - repeated ColumnStats column_stats = 4; -} - -message ColumnStats { - datafusion_common.ScalarValue min_value = 1; - datafusion_common.ScalarValue max_value = 2; - uint32 null_count = 3; - uint32 distinct_count = 4; + reserved 4; + reserved "column_stats"; } message OperatorMetricsSet { diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index 60f3debda4..09e41a38bf 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -561,8 +561,8 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| column with HyperLogLog while writing shuffle output, so the adaptive \ planner can estimate join cardinalities downstream of the shuffle. \ Costs hashing every key value on the write path and one sketch per \ - key column in each task's status update (4 KiB with the default \ - precision).".to_string(), + key column in each task's status update (at most about 2 KiB with \ + the default precision).".to_string(), DataType::Boolean, Some(true.to_string()), ), @@ -571,9 +571,9 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| "Precision of the HyperLogLog sketches that \ `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 \ (values outside that range are clamped). A sketch has \ - 2^precision one-byte registers and a standard error of about \ - 1.04 / 2^(precision / 2): 10 is 1 KiB and 3.3%, 12 is 4 KiB and \ - 1.6%, 14 is 16 KiB and 0.8%. The scheduler keeps one sketch per \ + 2^precision registers, sent at 4 bits each, and a standard error of \ + about 1.04 / 2^(precision / 2): 10 is 512 B and 3.3%, 12 is 2 KiB \ + and 1.6%, 14 is 8 KiB and 0.8%. The scheduler keeps one sketch per \ key column for every task of a stage, so lower it for stages with \ many thousands of tasks.".to_string(), DataType::UInt16, diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index f37bfde5f2..8d60900f3e 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -552,7 +552,6 @@ pub fn stats_for_partition( num_fields: usize, partition_locations: &[Vec], ) -> Result { - // TODO stats: add column statistics to PartitionStats let (num_rows, total_byte_size) = partition_locations .iter() .map(|location| { @@ -587,7 +586,6 @@ pub fn stats_for_partitions( num_fields: usize, partition_stats: impl Iterator, ) -> Statistics { - // TODO stats: add column statistics to PartitionStats let (num_rows, total_byte_size) = partition_stats.fold((Some(0), Some(0)), |(num_rows, total_byte_size), part| { // if any statistic is unkown it makes the entire statistic unkown diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index 57ab3aaa73..c2606e4138 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -36,7 +36,7 @@ use crate::execution_plans::{ use crate::extension::SessionConfigExt; use crate::utils; -use crate::hll::HyperLogLog; +use crate::hll::{HyperLogLog, HyperLogLogUnion}; use crate::serde::protobuf::{ ColumnSketch, ShuffleWritePartition, SketchType, TaskColumnStats, }; @@ -261,7 +261,7 @@ pub(crate) struct ColumnStatsCollector(Arc>); struct ColumnStatsTotals { null_counts: Vec, /// `(column index, sketch)` for each sketched column. - sketches: Vec<(usize, HyperLogLog)>, + sketches: Vec<(usize, HyperLogLogUnion)>, } /// The columns a writer sketches: the plain column references among the hash @@ -293,7 +293,7 @@ impl ColumnStatsCollector { null_counts: vec![0; num_columns], sketches: sketched .iter() - .map(|&column| (column, HyperLogLog::new(precision))) + .map(|&column| (column, HyperLogLogUnion::new(precision))) .collect(), }; } @@ -355,7 +355,7 @@ impl ColumnStatsCollector { if let Some(entry) = stats.get_mut(*column) { entry.sketches.push(ColumnSketch { r#type: SketchType::Hll as i32, - data: sketch.as_bytes().to_vec(), + data: sketch.to_sketch().to_bytes(), }); } } @@ -407,9 +407,7 @@ impl Drop for ColumnStatsStream { } for ((_, slot), (_, sketch)) in total.sketches.iter_mut().zip(&self.sketches) { - // Both were made at the precision `reset` chose, so they merge. - let merged = slot.merge(sketch); - debug_assert!(merged, "stream sketch and total differ in precision"); + slot.update(sketch); } } } @@ -1715,10 +1713,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << crate::hll::DEFAULT_PRECISION); - let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data) - .unwrap() - .count(); + let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), crate::hll::DEFAULT_PRECISION); + let distinct = hll.count(); assert!((95..=105).contains(&distinct), "estimated {distinct}"); assert!(stats[1].sketches.is_empty()); Ok(()) @@ -1739,10 +1736,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << 10); - let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data) - .unwrap() - .count(); + let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), 10); + let distinct = hll.count(); // Standard error is 3.3% here. assert!((80..=120).contains(&distinct), "estimated {distinct}"); Ok(()) diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs index c6e797dfa2..6e205601e4 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs @@ -2107,8 +2107,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << 10); - let distinct = HyperLogLog::from_bytes(&sketch.data).unwrap().count(); + let hll = HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), 10); + let distinct = hll.count(); // 1500 distinct values at a standard error of 3.3%. assert!((1200..=1800).contains(&distinct), "estimated {distinct}"); Ok(()) diff --git a/ballista/core/src/hll.rs b/ballista/core/src/hll.rs index c8f2f57423..582299729f 100644 --- a/ballista/core/src/hll.rs +++ b/ballista/core/src/hll.rs @@ -15,32 +15,31 @@ // specific language governing permissions and limitations // under the License. -//! HyperLogLog distinct-count sketch for shuffle output columns. +//! HyperLogLog distinct-count sketch for shuffle output columns, backed by the +//! Apache DataSketches HLL sketch. //! -//! Shuffle writers update one sketch per sketched column, executors ship the -//! registers to the scheduler in `TaskColumnStats.sketches`, and the scheduler -//! merges every task's sketch for a column into a stage-level distinct count. +//! Shuffle writers update one sketch per sketched column, executors ship it to +//! the scheduler in `TaskColumnStats.sketches`, and the scheduler merges every +//! task's sketch for a column into a stage-level distinct count with a +//! [`HyperLogLogUnion`]. //! //! Values are hashed with DataFusion's `HLL_RANDOM_STATE`, the hash state -//! `approx_distinct` uses, and the count uses the same estimator (Ertl, -//! "New cardinality estimation algorithms for HyperLogLog sketches", 2017). +//! `approx_distinct` uses, and the hashes are added to the sketch. //! //! # Precision //! //! A sketch's precision `p` is the number of hash bits that pick a register. -//! It has `2^p` one-byte registers, which is also its serialized size, and a -//! standard error of about `1.04 / 2^(p/2)`, so each 4x in size halves the -//! error: +//! It has `2^p` registers, serialized at 4 bits each once it holds many +//! values (and much smaller before that), and a standard error of about +//! `1.04 / 2^(p/2)`, so each 4x in size halves the error: //! -//! | precision | size | standard error | -//! | --------- | ------ | -------------- | -//! | 10 | 1 KiB | 3.3% | -//! | 12 | 4 KiB | 1.6% | -//! | 14 | 16 KiB | 0.8% | +//! | precision | size | standard error | +//! | --------- | ------- | -------------- | +//! | 10 | 512 B | 3.3% | +//! | 12 | 2 KiB | 1.6% | +//! | 14 | 8 KiB | 0.8% | //! -//! The sketch is not tied to one precision on the wire: the register count -//! tells [`HyperLogLog::from_bytes`](crate::hll::HyperLogLog::from_bytes) which -//! one it holds, and only sketches of the same precision merge. +//! Sketches of different precisions merge, at the lowest of them. //! //! The default, [`DEFAULT_PRECISION`](crate::hll::DEFAULT_PRECISION), is 12 //! rather than DataFusion's 14. The estimate feeds join cardinalities, where @@ -48,14 +47,14 @@ //! estimated row count, so an error of a couple of percent is small next to //! the row-count errors that having a distinct count at all corrects. The //! cost of a sketch is its size, once per key column per task: the scheduler -//! keeps every successful task's sketches. Precision 10 would save 3 KiB of -//! that, about 60 MB for a stage of 10,000 tasks with two key columns. -//! Deployments that big can lower the precision with +//! keeps every successful task's sketches. Deployments with many thousands of +//! tasks per stage can lower the precision with //! `ballista.shuffle.collect_distinct_counts_precision`. use datafusion::arrow::array::Array; use datafusion::common::Result; use datafusion::common::hash_utils::{HLL_RANDOM_STATE, create_hashes}; +use datasketches::hll::{HllSketch, HllType, HllUnion}; /// The smallest supported precision, 16 registers. pub const MIN_PRECISION: u8 = 4; @@ -64,13 +63,9 @@ pub const MAX_PRECISION: u8 = 16; /// The precision shuffle writers use unless configured otherwise. pub const DEFAULT_PRECISION: u8 = 12; -/// A HyperLogLog sketch with `2^precision` one-byte registers. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct HyperLogLog { - /// Number of hash bits that select a register. - precision: u8, - registers: Box<[u8]>, -} +/// A HyperLogLog sketch with `2^precision` registers. +#[derive(Clone, Debug, PartialEq)] +pub struct HyperLogLog(HllSketch); impl HyperLogLog { /// An empty sketch with `2^precision` registers. @@ -79,46 +74,27 @@ impl HyperLogLog { /// /// If `precision` is outside [`MIN_PRECISION`] to [`MAX_PRECISION`]. pub fn new(precision: u8) -> Self { - assert!( - (MIN_PRECISION..=MAX_PRECISION).contains(&precision), - "HyperLogLog precision {precision} is outside \ - {MIN_PRECISION}..={MAX_PRECISION}" - ); - Self { - precision, - registers: vec![0; 1 << precision].into_boxed_slice(), - } + assert_precision(precision); + Self(HllSketch::new(precision, HllType::Hll8).expect("precision is in range")) } /// The number of hash bits that select a register. pub fn precision(&self) -> u8 { - self.precision + self.0.lg_config_k() } - /// Restores a sketch from [`Self::as_bytes`]. Returns `None` if `bytes` - /// is not a valid register array: its length must be `2^p` for a - /// supported precision `p`, and no register can exceed the largest rank - /// a hash can produce. + /// Restores a sketch from [`Self::to_bytes`]. Returns `None` if `bytes` + /// is not a sketch of a supported precision. pub fn from_bytes(bytes: &[u8]) -> Option { - if !bytes.len().is_power_of_two() { - return None; - } - let precision = u8::try_from(bytes.len().trailing_zeros()).ok()?; - if !(MIN_PRECISION..=MAX_PRECISION).contains(&precision) { - return None; - } - if bytes.iter().any(|&r| r > max_rank(precision)) { - return None; - } - Some(Self { - precision, - registers: bytes.into(), - }) + let sketch = HllSketch::deserialize(bytes).ok()?; + (MIN_PRECISION..=MAX_PRECISION) + .contains(&sketch.lg_config_k()) + .then_some(Self(sketch)) } - /// The registers, which are the sketch's serialized form. - pub fn as_bytes(&self) -> &[u8] { - &self.registers + /// The sketch's serialized form. + pub fn to_bytes(&self) -> Vec { + self.0.serialize() } /// Adds every non-null value of `array`. `hashes` is scratch space, @@ -128,11 +104,11 @@ impl HyperLogLog { hashes.resize(array.len(), 0); create_hashes([array], &HLL_RANDOM_STATE, hashes)?; match array.logical_nulls() { - None => hashes.iter().for_each(|&hash| self.add_hashed(hash)), + None => hashes.iter().for_each(|&hash| self.0.update(hash)), Some(nulls) => { for (row, &hash) in hashes.iter().enumerate() { if nulls.is_valid(row) { - self.add_hashed(hash); + self.0.update(hash); } } } @@ -140,85 +116,57 @@ impl HyperLogLog { Ok(()) } - #[inline] - fn add_hashed(&mut self, hash: u64) { - let index = (hash & ((1 << self.precision) - 1)) as usize; - let rho = ((hash >> self.precision) | (1 << (64 - self.precision))) - .trailing_zeros() as u8 - + 1; - self.registers[index] = self.registers[index].max(rho); - } - - /// Folds `other` into this sketch, so it counts the union of both inputs. - /// Registers of different precisions don't line up, so if the two differ - /// this returns `false` and leaves the sketch unchanged. - #[must_use = "a sketch of a different precision is not merged"] - pub fn merge(&mut self, other: &HyperLogLog) -> bool { - if self.precision != other.precision { - return false; - } - for (mine, theirs) in self.registers.iter_mut().zip(other.registers.iter()) { - *mine = (*mine).max(*theirs); - } - true - } - /// The estimated number of distinct values added. pub fn count(&self) -> usize { - let q = 64 - self.precision as usize; - let mut histogram = [0u32; 64 - MIN_PRECISION as usize + 2]; - for &r in self.registers.iter() { - histogram[r as usize] += 1; - } - let m = self.registers.len() as f64; - let mut z = m * tau((m - histogram[q + 1] as f64) / m); - for &h in histogram[1..=q].iter().rev() { - z += h as f64; - z *= 0.5; - } - z += m * sigma(histogram[0] as f64 / m); - (0.5 / 2_f64.ln() * m * m / z).round() as usize + self.0.estimate().round() as usize } } -/// The largest value a register can hold at `precision`: the rank of a hash -/// whose bits after the register index are all zero. -fn max_rank(precision: u8) -> u8 { - 64 - precision + 1 -} +/// Merges many [`HyperLogLog`] sketches into one. +#[derive(Clone, Debug)] +pub struct HyperLogLogUnion(HllUnion); + +impl HyperLogLogUnion { + /// An empty union of at most `2^precision` registers. Sketches of a + /// higher precision are merged at this one. + /// + /// # Panics + /// + /// If `precision` is outside [`MIN_PRECISION`] to [`MAX_PRECISION`]. + pub fn new(precision: u8) -> Self { + assert_precision(precision); + Self(HllUnion::new(precision).expect("precision is in range")) + } -fn sigma(x: f64) -> f64 { - if x == 1. { - return f64::INFINITY; + /// The highest precision the union keeps. + pub fn precision(&self) -> u8 { + self.0.lg_max_k() } - let (mut x, mut y, mut z) = (x, 1.0, x); - loop { - x *= x; - let z_prime = z; - z += x * y; - y += y; - if z_prime == z { - return z; - } + + /// Folds `sketch` into the union. + pub fn update(&mut self, sketch: &HyperLogLog) { + self.0.update(&sketch.0); } -} -fn tau(x: f64) -> f64 { - if x == 0. || x == 1. { - return 0.; + /// The estimated number of distinct values across the merged sketches. + pub fn count(&self) -> usize { + self.0.estimate().round() as usize } - let (mut x, mut y, mut z) = (x, 1.0, 1.0 - x); - loop { - x = x.sqrt(); - let z_prime = z; - y *= 0.5; - z -= (1.0 - x).powi(2) * y; - if z_prime == z { - return z / 3.0; - } + + /// The union as a sketch, in its most compact form. + pub fn to_sketch(&self) -> HyperLogLog { + HyperLogLog(self.0.to_sketch(HllType::Hll4)) } } +fn assert_precision(precision: u8) { + assert!( + (MIN_PRECISION..=MAX_PRECISION).contains(&precision), + "HyperLogLog precision {precision} is outside \ + {MIN_PRECISION}..={MAX_PRECISION}" + ); +} + #[cfg(test)] mod tests { use super::*; @@ -235,6 +183,12 @@ mod tests { sketch_at(DEFAULT_PRECISION, values) } + fn union_of(precision: u8, sketches: &[HyperLogLog]) -> HyperLogLogUnion { + let mut union = HyperLogLogUnion::new(precision); + sketches.iter().for_each(|sketch| union.update(sketch)); + union + } + // Six standard errors, so the tests are deterministic in practice. fn assert_close(precision: u8, got: usize, expected: usize) { let registers = (1u64 << precision) as f64; @@ -275,12 +229,14 @@ mod tests { } #[test] - fn size_is_two_to_the_precision() { - for precision in MIN_PRECISION..=MAX_PRECISION { - let hll = HyperLogLog::new(precision); - assert_eq!(hll.precision(), precision); - assert_eq!(hll.as_bytes().len(), 1 << precision); - } + fn few_distinct_values_serialize_small() { + let few = sketch([1, 2, 3]).to_bytes().len(); + let many = union_of(DEFAULT_PRECISION, &[sketch(0..1_000_000)]) + .to_sketch() + .to_bytes() + .len(); + assert!(few < 100, "{few} bytes"); + assert!(many <= (1 << DEFAULT_PRECISION) / 2 + 100, "{many} bytes"); } #[test] @@ -292,45 +248,45 @@ mod tests { } #[test] - fn merge_counts_the_union() { + fn union_counts_the_union() { for precision in [10, DEFAULT_PRECISION] { - let mut left = sketch_at(precision, 0..60_000); - assert!(left.merge(&sketch_at(precision, 40_000..100_000))); - assert_close(precision, left.count(), 100_000); + let union = union_of( + precision, + &[ + sketch_at(precision, 0..60_000), + sketch_at(precision, 40_000..100_000), + ], + ); + assert_close(precision, union.count(), 100_000); + assert_close(precision, union.to_sketch().count(), 100_000); } } #[test] - fn merge_refuses_a_different_precision() { - let mut left = sketch_at(12, 0..1_000); - let before = left.clone(); - assert!(!left.merge(&sketch_at(10, 0..5_000))); - assert_eq!(left, before); + fn union_merges_different_precisions_at_the_lowest() { + let union = union_of( + DEFAULT_PRECISION, + &[sketch_at(12, 0..60_000), sketch_at(10, 40_000..100_000)], + ); + assert_eq!(union.to_sketch().precision(), 10); + assert_close(10, union.count(), 100_000); } #[test] fn round_trips_through_bytes() { for precision in [MIN_PRECISION, 10, DEFAULT_PRECISION, MAX_PRECISION] { let hll = sketch_at(precision, 0..1_000); - assert_eq!(HyperLogLog::from_bytes(hll.as_bytes()), Some(hll)); + let restored = HyperLogLog::from_bytes(&hll.to_bytes()).unwrap(); + assert_eq!(restored.precision(), precision); + assert_eq!(restored.count(), hll.count()); } } #[test] fn from_bytes_rejects_what_is_not_a_sketch() { - // Not a supported register count: empty, not a power of two, below - // the smallest precision, above the largest. - for len in [0, 1, 8, 15, 3_000, 1 << 17] { - assert_eq!(HyperLogLog::from_bytes(&vec![0; len]), None, "len {len}"); - } - // A register above the largest rank its precision allows. - for precision in [MIN_PRECISION, DEFAULT_PRECISION, MAX_PRECISION] { - let mut bytes = vec![0; 1 << precision]; - bytes[0] = max_rank(precision); - assert!(HyperLogLog::from_bytes(&bytes).is_some()); - bytes[0] += 1; - assert_eq!(HyperLogLog::from_bytes(&bytes), None); - } + assert_eq!(HyperLogLog::from_bytes(&[]), None); + assert_eq!(HyperLogLog::from_bytes(&[1, 2, 3]), None); + assert_eq!(HyperLogLog::from_bytes(&vec![0; 4096]), None); } #[test] diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index 41491eb2a8..57e772cb9e 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -1039,7 +1039,7 @@ pub struct TaskId { pub vcores_consumed: u32, } #[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct PartitionStats { #[prost(int64, tag = "1")] pub num_rows: i64, @@ -1047,20 +1047,6 @@ pub struct PartitionStats { pub num_batches: i64, #[prost(int64, tag = "3")] pub num_bytes: i64, - #[prost(message, repeated, tag = "4")] - pub column_stats: ::prost::alloc::vec::Vec, -} -#[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnStats { - #[prost(message, optional, tag = "1")] - pub min_value: ::core::option::Option<::datafusion_proto_common::ScalarValue>, - #[prost(message, optional, tag = "2")] - pub max_value: ::core::option::Option<::datafusion_proto_common::ScalarValue>, - #[prost(uint32, tag = "3")] - pub null_count: u32, - #[prost(uint32, tag = "4")] - pub distinct_count: u32, } #[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/ballista/core/src/serde/scheduler/to_proto.rs b/ballista/core/src/serde/scheduler/to_proto.rs index d2dbca71f6..87486c5204 100644 --- a/ballista/core/src/serde/scheduler/to_proto.rs +++ b/ballista/core/src/serde/scheduler/to_proto.rs @@ -138,7 +138,6 @@ impl Into for PartitionStats { num_rows: self.num_rows.map(|n| n as i64).unwrap_or(none_value), num_batches: self.num_batches.map(|n| n as i64).unwrap_or(none_value), num_bytes: self.num_bytes.map(|n| n as i64).unwrap_or(none_value), - column_stats: vec![], } } } diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index a38c7c0f41..d4f97557b0 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -38,7 +38,7 @@ use ballista_core::execution_plans::{ PipelinedShuffleWriterExec, RangeShuffleWriterExec, ShuffleReaderExec, ShuffleWriterExec, SortShuffleWriterExec, TaskRuntimeStats, TaskWindowState, }; -use ballista_core::hll::HyperLogLog; +use ballista_core::hll::{HyperLogLog, HyperLogLogUnion}; use ballista_core::serde::protobuf::failed_task::FailedReason; use ballista_core::serde::protobuf::{ FailedTask, OperatorMetricsSet, ResultLost, RuntimeStatsReport, SketchType, @@ -648,9 +648,9 @@ impl Debug for ResolvedStage { enum MergedSketch { /// No successful task has been added yet. Empty, - Merged(HyperLogLog), - /// Some task didn't sketch the column, or sketched it at a different - /// precision from the others, so the stage has no distinct count for it. + Merged(HyperLogLogUnion), + /// Some task didn't sketch the column, so the stage has no distinct count + /// for it. Unavailable, } @@ -659,13 +659,14 @@ impl MergedSketch { /// the task didn't make one. fn add(&mut self, sketch: Option) { *self = match (std::mem::replace(self, Self::Unavailable), sketch) { - (Self::Empty, Some(sketch)) => Self::Merged(sketch), - (Self::Merged(mut merged), Some(sketch)) => { - if merged.merge(&sketch) { - Self::Merged(merged) - } else { - Self::Unavailable - } + (Self::Empty, Some(sketch)) => { + let mut union = HyperLogLogUnion::new(sketch.precision()); + union.update(&sketch); + Self::Merged(union) + } + (Self::Merged(mut union), Some(sketch)) => { + union.update(&sketch); + Self::Merged(union) } _ => Self::Unavailable, }; @@ -730,7 +731,7 @@ impl RunningStage { /// wrote, one entry per output-schema column. `null_count` is `Exact` only /// when every successful task reported that column. `distinct_count` is an /// `Inexact` estimate from the tasks' merged HLL sketches, and only set - /// when every successful task sketched that column at the same precision. + /// when every successful task sketched that column. pub fn output_column_stats(&self) -> Vec { let num_columns = self.plan.schema().fields().len(); let mut output_column_stats = vec![ @@ -774,8 +775,8 @@ impl RunningStage { } } for (slot, sketch) in output_column_stats.iter_mut().zip(sketches) { - if let MergedSketch::Merged(sketch) = sketch { - slot.distinct_count = Precision::Inexact(sketch.count()); + if let MergedSketch::Merged(union) = sketch { + slot.distinct_count = Precision::Inexact(union.count()); } } output_column_stats @@ -1805,7 +1806,7 @@ mod tests { null_count: 0, sketches: vec![ColumnSketch { r#type: SketchType::Hll as i32, - data: hll.as_bytes().to_vec(), + data: hll.to_bytes(), }], } } @@ -1854,9 +1855,8 @@ mod tests { } #[test] - fn test_to_successful_leaves_distinct_count_absent_if_precisions_differ() { - // Registers of different precisions can't be merged, in either order, - // and a later task of the first precision doesn't bring the count back. + fn test_to_successful_merges_sketches_of_different_precisions() { + // The union merges at the lowest precision, in any order. for precisions in [[10, 12, 10], [12, 10, 12], [12, 12, 10]] { let stage = stage_with_col_stats( precisions @@ -1867,11 +1867,13 @@ mod tests { .collect(), ); let successful = stage.to_successful(); - assert_eq!( - successful.output_column_stats[0].distinct_count, - Precision::Absent, - "precisions {precisions:?}" - ); + let Precision::Inexact(distinct) = + successful.output_column_stats[0].distinct_count + else { + panic!("precisions {precisions:?}: expected an inexact distinct count"); + }; + // Standard error is 3.3% at precision 10. + assert!((800..=1200).contains(&distinct), "estimated {distinct}"); } } From e78c2a83d44fe201b12db6dae7b14544be526ea3 Mon Sep 17 00:00:00 2001 From: Bhargava Vadlamani Date: Tue, 17 Nov 2026 11:56:41 -0800 Subject: [PATCH 2/2] setup datasketches and update hll impl --- Cargo.lock | 4 ++-- ballista/core/src/hll.rs | 5 +++-- ballista/core/src/serde/generated/ballista.rs | 2 +- benchmarks/src/bin/shuffle_bench.rs | 2 +- docs/source/upgrading/56.0.0.md | 8 ++++++++ docs/source/user-guide/configs.md | 4 ++-- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac8e0b9bf1..05d489af8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5583,7 +5583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5604,7 +5604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad9c5423459fa2f729b8d9235fab810aa29628b171467744fc86d21460ad70d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/ballista/core/src/hll.rs b/ballista/core/src/hll.rs index 582299729f..c3a921e52c 100644 --- a/ballista/core/src/hll.rs +++ b/ballista/core/src/hll.rs @@ -23,8 +23,9 @@ //! task's sketch for a column into a stage-level distinct count with a //! [`HyperLogLogUnion`]. //! -//! Values are hashed with DataFusion's `HLL_RANDOM_STATE`, the hash state -//! `approx_distinct` uses, and the hashes are added to the sketch. +//! Values are hashed with DataFusion's `HLL_RANDOM_STATE` and the sketch hashes +//! each of those hashes again, so the hash state doesn't decide the result and +//! the sketch is not compatible with DataFusion's `approx_distinct`. //! //! # Precision //! diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index 1d51e9f088..6587e1f57b 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -1038,7 +1038,7 @@ pub struct FetchPipelinedPartition { pub registration_timeout_ms: u64, } #[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PartitionLocation { /// partition_id of the map stage who produces the shuffle. #[prost(uint32, tag = "1")] diff --git a/benchmarks/src/bin/shuffle_bench.rs b/benchmarks/src/bin/shuffle_bench.rs index 552a359539..080c872532 100644 --- a/benchmarks/src/bin/shuffle_bench.rs +++ b/benchmarks/src/bin/shuffle_bench.rs @@ -120,7 +120,7 @@ struct Args { collect_distinct_counts: bool, /// Precision of the distinct-count sketches: each has 2^precision - /// one-byte registers. + /// registers. #[arg(long, default_value_t = ballista_core::hll::DEFAULT_PRECISION)] distinct_count_precision: u8, } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 46aa4971d7..3d3aad8428 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -362,6 +362,14 @@ its `all_default_scalar_functions()`, `all_default_aggregate_functions()` and The scheduler's REST endpoint for its state, and the scheduler info popup of the TUI, no longer report a `spark_support` field. +#### `PartitionStats.column_stats` and `ColumnStats` are removed + +The protobuf `PartitionStats` message no longer has a `column_stats` field, and +the `ColumnStats` message is removed. Neither was ever populated; field number 4 +is reserved. Code that builds a `ballista_core::serde::protobuf::PartitionStats` +with a struct literal must drop `column_stats`. Per-column statistics are +reported in `SuccessfulTask.task_column_stats`. + ### Python client changes #### Summary diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 360d276c1e..23f15f1157 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -175,8 +175,8 @@ explanation of the sort-based shuffle writer. | key | type | default | description | | -------------------------------------------------------- | ------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ballista.shuffle.cache_broadcast_reads | Boolean | true | Lets the tasks on an executor share one fetch of each broadcast shuffle read (the build side of a broadcast hash join): the first task fetches and decodes it, the others reuse the batches. Without it every task fetches the whole build side, which at high partition counts multiplies the fetches by the number of tasks. Set to false to fetch per task. | -| ballista.shuffle.collect_distinct_counts | Boolean | true | Sketch the approximate distinct count of each hash-partitioning key column with HyperLogLog while writing shuffle output, so the adaptive planner can estimate join cardinalities downstream of the shuffle. Costs hashing every key value on the write path and one sketch per key column in each task's status update (4 KiB with the default precision). | -| ballista.shuffle.collect_distinct_counts_precision | UInt16 | 12 | Precision of the HyperLogLog sketches that `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 (values outside that range are clamped). A sketch has 2^precision one-byte registers and a standard error of about 1.04 / 2^(precision / 2): 10 is 1 KiB and 3.3%, 12 is 4 KiB and 1.6%, 14 is 16 KiB and 0.8%. The scheduler keeps one sketch per key column for every task of a stage, so lower it for stages with many thousands of tasks. | +| ballista.shuffle.collect_distinct_counts | Boolean | true | Sketch the approximate distinct count of each hash-partitioning key column with HyperLogLog while writing shuffle output, so the adaptive planner can estimate join cardinalities downstream of the shuffle. Costs hashing every key value on the write path and one sketch per key column in each task's status update (at most about 2 KiB with the default precision). | +| ballista.shuffle.collect_distinct_counts_precision | UInt16 | 12 | Precision of the HyperLogLog sketches that `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 (values outside that range are clamped). A sketch has 2^precision registers, sent at 4 bits each, and a standard error of about 1.04 / 2^(precision / 2): 10 is 512 B and 3.3%, 12 is 2 KiB and 1.6%, 14 is 8 KiB and 0.8%. The scheduler keeps one sketch per key column for every task of a stage, so lower it for stages with many thousands of tasks. | | ballista.shuffle.compression.codec | Utf8 | zstd | Compression codec used for shuffle data written to disk and sent between executors. Possible values: none, lz4, zstd. zstd (level 1) writes shuffle files about half the size of lz4 for little extra CPU, which matters most when shuffles cross the network | | ballista.shuffle.force_remote_read | Boolean | false | Forces the shuffle reader to always read partitions via the Arrow Flight client, even when partitions are local to the node. | | ballista.shuffle.max_concurrent_read_requests | UInt64 | 64 | Maximum concurrent requests shuffle reader can process |