diff --git a/Cargo.lock b/Cargo.lock index 7248d6f340..05d489af8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1136,6 +1136,7 @@ dependencies = [ "datafusion", "datafusion-proto", "datafusion-proto-common", + "datasketches", "futures", "log", "lru 0.19.0", @@ -3091,6 +3092,12 @@ dependencies = [ "url", ] +[[package]] +name = "datasketches" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17518a0e88b8a18c546a855d9db6455caaea7865774364bff759edf6a4731aca" + [[package]] name = "defmt" version = "1.1.1" diff --git a/Cargo.toml b/Cargo.toml index c0c825d7d8..3c89cc4091 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ datafusion-proto-common = { git = "https://github.com/apache/datafusion", tag = datafusion-substrait = { git = "https://github.com/apache/datafusion", tag = "56.0.0-rc1" } ctor = { version = "1.0" } +datasketches = { version = "0.6", default-features = false, features = ["hll"] } insta = "1.47" itertools = "0.15" mimalloc = { version = "0.1" } diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 28704b122c..be9f0fc085 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -51,6 +51,7 @@ clap = { workspace = true, optional = true } datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-proto-common = { workspace = true } +datasketches = { workspace = true } futures = { workspace = true } log = { workspace = true } lru = { version = "0.19", optional = true } diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto index 17174d45b0..a598657f2f 100644 --- a/ballista/core/proto/ballista.proto +++ b/ballista/core/proto/ballista.proto @@ -780,14 +780,8 @@ message PartitionStats { int64 num_rows = 1; int64 num_batches = 2; int64 num_bytes = 3; - repeated ColumnStats column_stats = 4; -} - -message ColumnStats { - datafusion_common.ScalarValue min_value = 1; - datafusion_common.ScalarValue max_value = 2; - uint32 null_count = 3; - uint32 distinct_count = 4; + reserved 4; + reserved "column_stats"; } message OperatorMetricsSet { diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index d40c31b64a..c617e3cd45 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -612,8 +612,8 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| column with HyperLogLog while writing shuffle output, so the adaptive \ planner can estimate join cardinalities downstream of the shuffle. \ Costs hashing every key value on the write path and one sketch per \ - key column in each task's status update (4 KiB with the default \ - precision).".to_string(), + key column in each task's status update (at most about 2 KiB with \ + the default precision).".to_string(), DataType::Boolean, Some(true.to_string()), ), @@ -622,9 +622,9 @@ static CONFIG_ENTRIES: LazyLock> = LazyLock::new(|| "Precision of the HyperLogLog sketches that \ `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 \ (values outside that range are clamped). A sketch has \ - 2^precision one-byte registers and a standard error of about \ - 1.04 / 2^(precision / 2): 10 is 1 KiB and 3.3%, 12 is 4 KiB and \ - 1.6%, 14 is 16 KiB and 0.8%. The scheduler keeps one sketch per \ + 2^precision registers, sent at 4 bits each, and a standard error of \ + about 1.04 / 2^(precision / 2): 10 is 512 B and 3.3%, 12 is 2 KiB \ + and 1.6%, 14 is 8 KiB and 0.8%. The scheduler keeps one sketch per \ key column for every task of a stage, so lower it for stages with \ many thousands of tasks.".to_string(), DataType::UInt16, diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs b/ballista/core/src/execution_plans/shuffle_reader.rs index 51ebfb74ee..0cb46d1c96 100644 --- a/ballista/core/src/execution_plans/shuffle_reader.rs +++ b/ballista/core/src/execution_plans/shuffle_reader.rs @@ -619,7 +619,6 @@ pub fn stats_for_partition( num_fields: usize, partition_locations: &[Vec], ) -> Result { - // TODO stats: add column statistics to PartitionStats let (num_rows, total_byte_size) = partition_locations .iter() .map(|location| { @@ -654,7 +653,6 @@ pub fn stats_for_partitions( num_fields: usize, partition_stats: impl Iterator, ) -> Statistics { - // TODO stats: add column statistics to PartitionStats let (num_rows, total_byte_size) = partition_stats.fold((Some(0), Some(0)), |(num_rows, total_byte_size), part| { // if any statistic is unkown it makes the entire statistic unkown diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs b/ballista/core/src/execution_plans/shuffle_writer.rs index b3a1c8a52e..b476a9b04c 100644 --- a/ballista/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/core/src/execution_plans/shuffle_writer.rs @@ -36,7 +36,7 @@ use crate::execution_plans::{ use crate::extension::SessionConfigExt; use crate::utils; -use crate::hll::HyperLogLog; +use crate::hll::{HyperLogLog, HyperLogLogUnion}; use crate::serde::protobuf::{ ColumnSketch, ShuffleWritePartition, SketchType, TaskColumnStats, }; @@ -259,7 +259,7 @@ pub(crate) struct ColumnStatsCollector(Arc>); struct ColumnStatsTotals { null_counts: Vec, /// `(column index, sketch)` for each sketched column. - sketches: Vec<(usize, HyperLogLog)>, + sketches: Vec<(usize, HyperLogLogUnion)>, } /// The columns a writer sketches: the plain column references among the hash @@ -291,7 +291,7 @@ impl ColumnStatsCollector { null_counts: vec![0; num_columns], sketches: sketched .iter() - .map(|&column| (column, HyperLogLog::new(precision))) + .map(|&column| (column, HyperLogLogUnion::new(precision))) .collect(), }; } @@ -360,7 +360,7 @@ impl ColumnStatsCollector { if let Some(entry) = stats.get_mut(*column) { entry.sketches.push(ColumnSketch { r#type: SketchType::Hll as i32, - data: sketch.as_bytes().to_vec(), + data: sketch.to_sketch().to_bytes(), }); } } @@ -420,9 +420,7 @@ impl Drop for ColumnStatsStream { } for ((_, slot), (_, sketch)) in total.sketches.iter_mut().zip(&self.sketches) { - // Both were made at the precision `reset` chose, so they merge. - let merged = slot.merge(sketch); - debug_assert!(merged, "stream sketch and total differ in precision"); + slot.update(sketch); } } } @@ -1680,10 +1678,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << crate::hll::DEFAULT_PRECISION); - let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data) - .unwrap() - .count(); + let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), crate::hll::DEFAULT_PRECISION); + let distinct = hll.count(); assert!((95..=105).contains(&distinct), "estimated {distinct}"); assert!(stats[1].sketches.is_empty()); Ok(()) @@ -1704,10 +1701,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << 10); - let distinct = crate::hll::HyperLogLog::from_bytes(&sketch.data) - .unwrap() - .count(); + let hll = crate::hll::HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), 10); + let distinct = hll.count(); // Standard error is 3.3% here. assert!((80..=120).contains(&distinct), "estimated {distinct}"); Ok(()) diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs b/ballista/core/src/execution_plans/sort_shuffle/writer.rs index a92e1aae94..fabeb1f1f7 100644 --- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs +++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs @@ -2331,8 +2331,9 @@ mod tests { let [sketch] = stats[0].sketches.as_slice() else { panic!("expected one sketch on the hash key, got {:?}", stats[0]); }; - assert_eq!(sketch.data.len(), 1 << 10); - let distinct = HyperLogLog::from_bytes(&sketch.data).unwrap().count(); + let hll = HyperLogLog::from_bytes(&sketch.data).unwrap(); + assert_eq!(hll.precision(), 10); + let distinct = hll.count(); // 1500 distinct values at a standard error of 3.3%. assert!((1200..=1800).contains(&distinct), "estimated {distinct}"); Ok(()) diff --git a/ballista/core/src/hll.rs b/ballista/core/src/hll.rs index c8f2f57423..c3a921e52c 100644 --- a/ballista/core/src/hll.rs +++ b/ballista/core/src/hll.rs @@ -15,32 +15,32 @@ // specific language governing permissions and limitations // under the License. -//! HyperLogLog distinct-count sketch for shuffle output columns. +//! HyperLogLog distinct-count sketch for shuffle output columns, backed by the +//! Apache DataSketches HLL sketch. //! -//! Shuffle writers update one sketch per sketched column, executors ship the -//! registers to the scheduler in `TaskColumnStats.sketches`, and the scheduler -//! merges every task's sketch for a column into a stage-level distinct count. +//! Shuffle writers update one sketch per sketched column, executors ship it to +//! the scheduler in `TaskColumnStats.sketches`, and the scheduler merges every +//! task's sketch for a column into a stage-level distinct count with a +//! [`HyperLogLogUnion`]. //! -//! Values are hashed with DataFusion's `HLL_RANDOM_STATE`, the hash state -//! `approx_distinct` uses, and the count uses the same estimator (Ertl, -//! "New cardinality estimation algorithms for HyperLogLog sketches", 2017). +//! Values are hashed with DataFusion's `HLL_RANDOM_STATE` and the sketch hashes +//! each of those hashes again, so the hash state doesn't decide the result and +//! the sketch is not compatible with DataFusion's `approx_distinct`. //! //! # Precision //! //! A sketch's precision `p` is the number of hash bits that pick a register. -//! It has `2^p` one-byte registers, which is also its serialized size, and a -//! standard error of about `1.04 / 2^(p/2)`, so each 4x in size halves the -//! error: +//! It has `2^p` registers, serialized at 4 bits each once it holds many +//! values (and much smaller before that), and a standard error of about +//! `1.04 / 2^(p/2)`, so each 4x in size halves the error: //! -//! | precision | size | standard error | -//! | --------- | ------ | -------------- | -//! | 10 | 1 KiB | 3.3% | -//! | 12 | 4 KiB | 1.6% | -//! | 14 | 16 KiB | 0.8% | +//! | precision | size | standard error | +//! | --------- | ------- | -------------- | +//! | 10 | 512 B | 3.3% | +//! | 12 | 2 KiB | 1.6% | +//! | 14 | 8 KiB | 0.8% | //! -//! The sketch is not tied to one precision on the wire: the register count -//! tells [`HyperLogLog::from_bytes`](crate::hll::HyperLogLog::from_bytes) which -//! one it holds, and only sketches of the same precision merge. +//! Sketches of different precisions merge, at the lowest of them. //! //! The default, [`DEFAULT_PRECISION`](crate::hll::DEFAULT_PRECISION), is 12 //! rather than DataFusion's 14. The estimate feeds join cardinalities, where @@ -48,14 +48,14 @@ //! estimated row count, so an error of a couple of percent is small next to //! the row-count errors that having a distinct count at all corrects. The //! cost of a sketch is its size, once per key column per task: the scheduler -//! keeps every successful task's sketches. Precision 10 would save 3 KiB of -//! that, about 60 MB for a stage of 10,000 tasks with two key columns. -//! Deployments that big can lower the precision with +//! keeps every successful task's sketches. Deployments with many thousands of +//! tasks per stage can lower the precision with //! `ballista.shuffle.collect_distinct_counts_precision`. use datafusion::arrow::array::Array; use datafusion::common::Result; use datafusion::common::hash_utils::{HLL_RANDOM_STATE, create_hashes}; +use datasketches::hll::{HllSketch, HllType, HllUnion}; /// The smallest supported precision, 16 registers. pub const MIN_PRECISION: u8 = 4; @@ -64,13 +64,9 @@ pub const MAX_PRECISION: u8 = 16; /// The precision shuffle writers use unless configured otherwise. pub const DEFAULT_PRECISION: u8 = 12; -/// A HyperLogLog sketch with `2^precision` one-byte registers. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct HyperLogLog { - /// Number of hash bits that select a register. - precision: u8, - registers: Box<[u8]>, -} +/// A HyperLogLog sketch with `2^precision` registers. +#[derive(Clone, Debug, PartialEq)] +pub struct HyperLogLog(HllSketch); impl HyperLogLog { /// An empty sketch with `2^precision` registers. @@ -79,46 +75,27 @@ impl HyperLogLog { /// /// If `precision` is outside [`MIN_PRECISION`] to [`MAX_PRECISION`]. pub fn new(precision: u8) -> Self { - assert!( - (MIN_PRECISION..=MAX_PRECISION).contains(&precision), - "HyperLogLog precision {precision} is outside \ - {MIN_PRECISION}..={MAX_PRECISION}" - ); - Self { - precision, - registers: vec![0; 1 << precision].into_boxed_slice(), - } + assert_precision(precision); + Self(HllSketch::new(precision, HllType::Hll8).expect("precision is in range")) } /// The number of hash bits that select a register. pub fn precision(&self) -> u8 { - self.precision + self.0.lg_config_k() } - /// Restores a sketch from [`Self::as_bytes`]. Returns `None` if `bytes` - /// is not a valid register array: its length must be `2^p` for a - /// supported precision `p`, and no register can exceed the largest rank - /// a hash can produce. + /// Restores a sketch from [`Self::to_bytes`]. Returns `None` if `bytes` + /// is not a sketch of a supported precision. pub fn from_bytes(bytes: &[u8]) -> Option { - if !bytes.len().is_power_of_two() { - return None; - } - let precision = u8::try_from(bytes.len().trailing_zeros()).ok()?; - if !(MIN_PRECISION..=MAX_PRECISION).contains(&precision) { - return None; - } - if bytes.iter().any(|&r| r > max_rank(precision)) { - return None; - } - Some(Self { - precision, - registers: bytes.into(), - }) + let sketch = HllSketch::deserialize(bytes).ok()?; + (MIN_PRECISION..=MAX_PRECISION) + .contains(&sketch.lg_config_k()) + .then_some(Self(sketch)) } - /// The registers, which are the sketch's serialized form. - pub fn as_bytes(&self) -> &[u8] { - &self.registers + /// The sketch's serialized form. + pub fn to_bytes(&self) -> Vec { + self.0.serialize() } /// Adds every non-null value of `array`. `hashes` is scratch space, @@ -128,11 +105,11 @@ impl HyperLogLog { hashes.resize(array.len(), 0); create_hashes([array], &HLL_RANDOM_STATE, hashes)?; match array.logical_nulls() { - None => hashes.iter().for_each(|&hash| self.add_hashed(hash)), + None => hashes.iter().for_each(|&hash| self.0.update(hash)), Some(nulls) => { for (row, &hash) in hashes.iter().enumerate() { if nulls.is_valid(row) { - self.add_hashed(hash); + self.0.update(hash); } } } @@ -140,85 +117,57 @@ impl HyperLogLog { Ok(()) } - #[inline] - fn add_hashed(&mut self, hash: u64) { - let index = (hash & ((1 << self.precision) - 1)) as usize; - let rho = ((hash >> self.precision) | (1 << (64 - self.precision))) - .trailing_zeros() as u8 - + 1; - self.registers[index] = self.registers[index].max(rho); - } - - /// Folds `other` into this sketch, so it counts the union of both inputs. - /// Registers of different precisions don't line up, so if the two differ - /// this returns `false` and leaves the sketch unchanged. - #[must_use = "a sketch of a different precision is not merged"] - pub fn merge(&mut self, other: &HyperLogLog) -> bool { - if self.precision != other.precision { - return false; - } - for (mine, theirs) in self.registers.iter_mut().zip(other.registers.iter()) { - *mine = (*mine).max(*theirs); - } - true - } - /// The estimated number of distinct values added. pub fn count(&self) -> usize { - let q = 64 - self.precision as usize; - let mut histogram = [0u32; 64 - MIN_PRECISION as usize + 2]; - for &r in self.registers.iter() { - histogram[r as usize] += 1; - } - let m = self.registers.len() as f64; - let mut z = m * tau((m - histogram[q + 1] as f64) / m); - for &h in histogram[1..=q].iter().rev() { - z += h as f64; - z *= 0.5; - } - z += m * sigma(histogram[0] as f64 / m); - (0.5 / 2_f64.ln() * m * m / z).round() as usize + self.0.estimate().round() as usize } } -/// The largest value a register can hold at `precision`: the rank of a hash -/// whose bits after the register index are all zero. -fn max_rank(precision: u8) -> u8 { - 64 - precision + 1 -} +/// Merges many [`HyperLogLog`] sketches into one. +#[derive(Clone, Debug)] +pub struct HyperLogLogUnion(HllUnion); + +impl HyperLogLogUnion { + /// An empty union of at most `2^precision` registers. Sketches of a + /// higher precision are merged at this one. + /// + /// # Panics + /// + /// If `precision` is outside [`MIN_PRECISION`] to [`MAX_PRECISION`]. + pub fn new(precision: u8) -> Self { + assert_precision(precision); + Self(HllUnion::new(precision).expect("precision is in range")) + } -fn sigma(x: f64) -> f64 { - if x == 1. { - return f64::INFINITY; + /// The highest precision the union keeps. + pub fn precision(&self) -> u8 { + self.0.lg_max_k() } - let (mut x, mut y, mut z) = (x, 1.0, x); - loop { - x *= x; - let z_prime = z; - z += x * y; - y += y; - if z_prime == z { - return z; - } + + /// Folds `sketch` into the union. + pub fn update(&mut self, sketch: &HyperLogLog) { + self.0.update(&sketch.0); } -} -fn tau(x: f64) -> f64 { - if x == 0. || x == 1. { - return 0.; + /// The estimated number of distinct values across the merged sketches. + pub fn count(&self) -> usize { + self.0.estimate().round() as usize } - let (mut x, mut y, mut z) = (x, 1.0, 1.0 - x); - loop { - x = x.sqrt(); - let z_prime = z; - y *= 0.5; - z -= (1.0 - x).powi(2) * y; - if z_prime == z { - return z / 3.0; - } + + /// The union as a sketch, in its most compact form. + pub fn to_sketch(&self) -> HyperLogLog { + HyperLogLog(self.0.to_sketch(HllType::Hll4)) } } +fn assert_precision(precision: u8) { + assert!( + (MIN_PRECISION..=MAX_PRECISION).contains(&precision), + "HyperLogLog precision {precision} is outside \ + {MIN_PRECISION}..={MAX_PRECISION}" + ); +} + #[cfg(test)] mod tests { use super::*; @@ -235,6 +184,12 @@ mod tests { sketch_at(DEFAULT_PRECISION, values) } + fn union_of(precision: u8, sketches: &[HyperLogLog]) -> HyperLogLogUnion { + let mut union = HyperLogLogUnion::new(precision); + sketches.iter().for_each(|sketch| union.update(sketch)); + union + } + // Six standard errors, so the tests are deterministic in practice. fn assert_close(precision: u8, got: usize, expected: usize) { let registers = (1u64 << precision) as f64; @@ -275,12 +230,14 @@ mod tests { } #[test] - fn size_is_two_to_the_precision() { - for precision in MIN_PRECISION..=MAX_PRECISION { - let hll = HyperLogLog::new(precision); - assert_eq!(hll.precision(), precision); - assert_eq!(hll.as_bytes().len(), 1 << precision); - } + fn few_distinct_values_serialize_small() { + let few = sketch([1, 2, 3]).to_bytes().len(); + let many = union_of(DEFAULT_PRECISION, &[sketch(0..1_000_000)]) + .to_sketch() + .to_bytes() + .len(); + assert!(few < 100, "{few} bytes"); + assert!(many <= (1 << DEFAULT_PRECISION) / 2 + 100, "{many} bytes"); } #[test] @@ -292,45 +249,45 @@ mod tests { } #[test] - fn merge_counts_the_union() { + fn union_counts_the_union() { for precision in [10, DEFAULT_PRECISION] { - let mut left = sketch_at(precision, 0..60_000); - assert!(left.merge(&sketch_at(precision, 40_000..100_000))); - assert_close(precision, left.count(), 100_000); + let union = union_of( + precision, + &[ + sketch_at(precision, 0..60_000), + sketch_at(precision, 40_000..100_000), + ], + ); + assert_close(precision, union.count(), 100_000); + assert_close(precision, union.to_sketch().count(), 100_000); } } #[test] - fn merge_refuses_a_different_precision() { - let mut left = sketch_at(12, 0..1_000); - let before = left.clone(); - assert!(!left.merge(&sketch_at(10, 0..5_000))); - assert_eq!(left, before); + fn union_merges_different_precisions_at_the_lowest() { + let union = union_of( + DEFAULT_PRECISION, + &[sketch_at(12, 0..60_000), sketch_at(10, 40_000..100_000)], + ); + assert_eq!(union.to_sketch().precision(), 10); + assert_close(10, union.count(), 100_000); } #[test] fn round_trips_through_bytes() { for precision in [MIN_PRECISION, 10, DEFAULT_PRECISION, MAX_PRECISION] { let hll = sketch_at(precision, 0..1_000); - assert_eq!(HyperLogLog::from_bytes(hll.as_bytes()), Some(hll)); + let restored = HyperLogLog::from_bytes(&hll.to_bytes()).unwrap(); + assert_eq!(restored.precision(), precision); + assert_eq!(restored.count(), hll.count()); } } #[test] fn from_bytes_rejects_what_is_not_a_sketch() { - // Not a supported register count: empty, not a power of two, below - // the smallest precision, above the largest. - for len in [0, 1, 8, 15, 3_000, 1 << 17] { - assert_eq!(HyperLogLog::from_bytes(&vec![0; len]), None, "len {len}"); - } - // A register above the largest rank its precision allows. - for precision in [MIN_PRECISION, DEFAULT_PRECISION, MAX_PRECISION] { - let mut bytes = vec![0; 1 << precision]; - bytes[0] = max_rank(precision); - assert!(HyperLogLog::from_bytes(&bytes).is_some()); - bytes[0] += 1; - assert_eq!(HyperLogLog::from_bytes(&bytes), None); - } + assert_eq!(HyperLogLog::from_bytes(&[]), None); + assert_eq!(HyperLogLog::from_bytes(&[1, 2, 3]), None); + assert_eq!(HyperLogLog::from_bytes(&vec![0; 4096]), None); } #[test] diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs index c72b90764a..6587e1f57b 100644 --- a/ballista/core/src/serde/generated/ballista.rs +++ b/ballista/core/src/serde/generated/ballista.rs @@ -1038,7 +1038,7 @@ pub struct FetchPipelinedPartition { pub registration_timeout_ms: u64, } #[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PartitionLocation { /// partition_id of the map stage who produces the shuffle. #[prost(uint32, tag = "1")] @@ -1103,7 +1103,7 @@ pub struct TaskId { pub vcores_consumed: u32, } #[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct PartitionStats { #[prost(int64, tag = "1")] pub num_rows: i64, @@ -1111,20 +1111,6 @@ pub struct PartitionStats { pub num_batches: i64, #[prost(int64, tag = "3")] pub num_bytes: i64, - #[prost(message, repeated, tag = "4")] - pub column_stats: ::prost::alloc::vec::Vec, -} -#[allow(clippy::all)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ColumnStats { - #[prost(message, optional, tag = "1")] - pub min_value: ::core::option::Option<::datafusion_proto_common::ScalarValue>, - #[prost(message, optional, tag = "2")] - pub max_value: ::core::option::Option<::datafusion_proto_common::ScalarValue>, - #[prost(uint32, tag = "3")] - pub null_count: u32, - #[prost(uint32, tag = "4")] - pub distinct_count: u32, } #[allow(clippy::all)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/ballista/core/src/serde/scheduler/to_proto.rs b/ballista/core/src/serde/scheduler/to_proto.rs index 333cf3017a..d65addf420 100644 --- a/ballista/core/src/serde/scheduler/to_proto.rs +++ b/ballista/core/src/serde/scheduler/to_proto.rs @@ -227,7 +227,6 @@ impl Into for PartitionStats { num_rows: self.num_rows.map(|n| n as i64).unwrap_or(none_value), num_batches: self.num_batches.map(|n| n as i64).unwrap_or(none_value), num_bytes: self.num_bytes.map(|n| n as i64).unwrap_or(none_value), - column_stats: vec![], } } } diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index 600738d596..292e3e084a 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -41,7 +41,7 @@ use ballista_core::execution_plans::{ ShuffleWriterExec, SortShuffleWriterExec, TaskRuntimeStats, TaskWindowState, UnresolvedShuffleExec, }; -use ballista_core::hll::HyperLogLog; +use ballista_core::hll::{HyperLogLog, HyperLogLogUnion}; use ballista_core::serde::protobuf::failed_task::FailedReason; use ballista_core::serde::protobuf::{ FailedTask, OperatorMetricsSet, ResultLost, RuntimeStatsReport, SketchType, @@ -659,9 +659,9 @@ impl Debug for ResolvedStage { enum MergedSketch { /// No successful task has been added yet. Empty, - Merged(HyperLogLog), - /// Some task didn't sketch the column, or sketched it at a different - /// precision from the others, so the stage has no distinct count for it. + Merged(HyperLogLogUnion), + /// Some task didn't sketch the column, so the stage has no distinct count + /// for it. Unavailable, } @@ -670,13 +670,14 @@ impl MergedSketch { /// the task didn't make one. fn add(&mut self, sketch: Option) { *self = match (std::mem::replace(self, Self::Unavailable), sketch) { - (Self::Empty, Some(sketch)) => Self::Merged(sketch), - (Self::Merged(mut merged), Some(sketch)) => { - if merged.merge(&sketch) { - Self::Merged(merged) - } else { - Self::Unavailable - } + (Self::Empty, Some(sketch)) => { + let mut union = HyperLogLogUnion::new(sketch.precision()); + union.update(&sketch); + Self::Merged(union) + } + (Self::Merged(mut union), Some(sketch)) => { + union.update(&sketch); + Self::Merged(union) } _ => Self::Unavailable, }; @@ -743,7 +744,7 @@ impl RunningStage { /// wrote, one entry per output-schema column. `null_count` is `Exact` only /// when every successful task reported that column. `distinct_count` is an /// `Inexact` estimate from the tasks' merged HLL sketches, and only set - /// when every successful task sketched that column at the same precision. + /// when every successful task sketched that column. pub fn output_column_stats(&self) -> Vec { let num_columns = self.plan.schema().fields().len(); let mut output_column_stats = vec![ @@ -787,8 +788,8 @@ impl RunningStage { } } for (slot, sketch) in output_column_stats.iter_mut().zip(sketches) { - if let MergedSketch::Merged(sketch) = sketch { - slot.distinct_count = Precision::Inexact(sketch.count()); + if let MergedSketch::Merged(union) = sketch { + slot.distinct_count = Precision::Inexact(union.count()); } } output_column_stats @@ -1956,7 +1957,7 @@ mod tests { null_count: 0, sketches: vec![ColumnSketch { r#type: SketchType::Hll as i32, - data: hll.as_bytes().to_vec(), + data: hll.to_bytes(), }], } } @@ -2005,9 +2006,8 @@ mod tests { } #[test] - fn test_to_successful_leaves_distinct_count_absent_if_precisions_differ() { - // Registers of different precisions can't be merged, in either order, - // and a later task of the first precision doesn't bring the count back. + fn test_to_successful_merges_sketches_of_different_precisions() { + // The union merges at the lowest precision, in any order. for precisions in [[10, 12, 10], [12, 10, 12], [12, 12, 10]] { let stage = stage_with_col_stats( precisions @@ -2018,11 +2018,13 @@ mod tests { .collect(), ); let successful = stage.to_successful(); - assert_eq!( - successful.output_column_stats[0].distinct_count, - Precision::Absent, - "precisions {precisions:?}" - ); + let Precision::Inexact(distinct) = + successful.output_column_stats[0].distinct_count + else { + panic!("precisions {precisions:?}: expected an inexact distinct count"); + }; + // Standard error is 3.3% at precision 10. + assert!((800..=1200).contains(&distinct), "estimated {distinct}"); } } diff --git a/benchmarks/src/bin/shuffle_bench.rs b/benchmarks/src/bin/shuffle_bench.rs index 552a359539..080c872532 100644 --- a/benchmarks/src/bin/shuffle_bench.rs +++ b/benchmarks/src/bin/shuffle_bench.rs @@ -120,7 +120,7 @@ struct Args { collect_distinct_counts: bool, /// Precision of the distinct-count sketches: each has 2^precision - /// one-byte registers. + /// registers. #[arg(long, default_value_t = ballista_core::hll::DEFAULT_PRECISION)] distinct_count_precision: u8, } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 46aa4971d7..3d3aad8428 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -362,6 +362,14 @@ its `all_default_scalar_functions()`, `all_default_aggregate_functions()` and The scheduler's REST endpoint for its state, and the scheduler info popup of the TUI, no longer report a `spark_support` field. +#### `PartitionStats.column_stats` and `ColumnStats` are removed + +The protobuf `PartitionStats` message no longer has a `column_stats` field, and +the `ColumnStats` message is removed. Neither was ever populated; field number 4 +is reserved. Code that builds a `ballista_core::serde::protobuf::PartitionStats` +with a struct literal must drop `column_stats`. Per-column statistics are +reported in `SuccessfulTask.task_column_stats`. + ### Python client changes #### Summary diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 360d276c1e..23f15f1157 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -175,8 +175,8 @@ explanation of the sort-based shuffle writer. | key | type | default | description | | -------------------------------------------------------- | ------- | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ballista.shuffle.cache_broadcast_reads | Boolean | true | Lets the tasks on an executor share one fetch of each broadcast shuffle read (the build side of a broadcast hash join): the first task fetches and decodes it, the others reuse the batches. Without it every task fetches the whole build side, which at high partition counts multiplies the fetches by the number of tasks. Set to false to fetch per task. | -| ballista.shuffle.collect_distinct_counts | Boolean | true | Sketch the approximate distinct count of each hash-partitioning key column with HyperLogLog while writing shuffle output, so the adaptive planner can estimate join cardinalities downstream of the shuffle. Costs hashing every key value on the write path and one sketch per key column in each task's status update (4 KiB with the default precision). | -| ballista.shuffle.collect_distinct_counts_precision | UInt16 | 12 | Precision of the HyperLogLog sketches that `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 (values outside that range are clamped). A sketch has 2^precision one-byte registers and a standard error of about 1.04 / 2^(precision / 2): 10 is 1 KiB and 3.3%, 12 is 4 KiB and 1.6%, 14 is 16 KiB and 0.8%. The scheduler keeps one sketch per key column for every task of a stage, so lower it for stages with many thousands of tasks. | +| ballista.shuffle.collect_distinct_counts | Boolean | true | Sketch the approximate distinct count of each hash-partitioning key column with HyperLogLog while writing shuffle output, so the adaptive planner can estimate join cardinalities downstream of the shuffle. Costs hashing every key value on the write path and one sketch per key column in each task's status update (at most about 2 KiB with the default precision). | +| ballista.shuffle.collect_distinct_counts_precision | UInt16 | 12 | Precision of the HyperLogLog sketches that `ballista.shuffle.collect_distinct_counts` collects, from 4 to 16 (values outside that range are clamped). A sketch has 2^precision registers, sent at 4 bits each, and a standard error of about 1.04 / 2^(precision / 2): 10 is 512 B and 3.3%, 12 is 2 KiB and 1.6%, 14 is 8 KiB and 0.8%. The scheduler keeps one sketch per key column for every task of a stage, so lower it for stages with many thousands of tasks. | | ballista.shuffle.compression.codec | Utf8 | zstd | Compression codec used for shuffle data written to disk and sent between executors. Possible values: none, lz4, zstd. zstd (level 1) writes shuffle files about half the size of lz4 for little extra CPU, which matters most when shuffles cross the network | | ballista.shuffle.force_remote_read | Boolean | false | Forces the shuffle reader to always read partitions via the Arrow Flight client, even when partitions are local to the node. | | ballista.shuffle.max_concurrent_read_requests | UInt64 | 64 | Maximum concurrent requests shuffle reader can process |