diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dcf066..9b4e894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ version; patch releases remain backward compatible. ## Unreleased +- `decode_binary_value` and `encode_binary_value` handle an `Array` or `Map` + run of zero-width `Tuple()` values in constant time per tuple node instead of + once per element. Before, a small nested value such as + `Array(Tuple(Tuple(Tuple())))` could drive work far beyond its input size. + The 1,000,000-element cap per container is unchanged. +- `decode_binary_value` returns `BinaryValueError::Invalid` for a caller-built + `FixedString(0)` or `QBit(T, 0)` type, matching `encode_binary_value`. - `ch-tcp-proto`: support compressed client-fed INSERT. Upload blocks and the empty upload terminator use LZ4 frames, with at most 1 MiB of plaintext per frame. Server schema blocks use the existing compressed receive path. diff --git a/CODEC_CONTRACT.md b/CODEC_CONTRACT.md index 9970fdc..8c5a13b 100644 --- a/CODEC_CONTRACT.md +++ b/CODEC_CONTRACT.md @@ -2490,6 +2490,23 @@ aggregate descriptor forms. The server writes SimpleAggregateFunction with `0x2e` and AggregateFunction with `0x25`; AggregateFunction value encoding remains unsupported. +A zero-width single value is `Tuple()` or a Tuple whose elements are all +zero-width, through aliases and LowCardinality. It has no value bytes, so an +`Array` or `Map` count of them is not backed by input. Both helpers keep the +1,000,000-element cap on such a count per container. Within the cap, +`decode_binary_value` appends the whole run in one step per tuple node, and +`encode_binary_value` writes the count after checking each child tuple length +once. An empty run does no work in either direction. Encode runs the per-row +checks from the first short row, or from the start when the run's columns carry +a tuple validity bitmap, a LowCardinality dictionary, or a shape mismatch, so +its errors match per-row encoding. `Nullable`, `Array`, `Map`, +and `Nothing` elements are not zero-width here: each value reads at least one +byte, and a non-null `Nothing` is `Unsupported`. `encode_binary_value` also +caps a run of `Nullable` zero-width values at 1,000,000, so a longer decoded +run does not encode again. `decode_binary_value` rejects +`FixedString(0)` and `QBit(T, 0)` as `Invalid`, since neither type parser can +produce them. + For a top-level SharedVariant or JSON shared cell, strip the top-level `Nullable(T)` to `T`, or `LowCardinality(Nullable(T))` to `LowCardinality(T)`, before writing both the descriptor and value. Route null rows as Dynamic NULL diff --git a/scripts/gen_fixtures.sh b/scripts/gen_fixtures.sh index fd3a67c..0b82ff1 100755 --- a/scripts/gen_fixtures.sh +++ b/scripts/gen_fixtures.sh @@ -535,6 +535,18 @@ FORMAT Native SQL ) +# Zero-width Array(Tuple()) values in Dynamic SharedVariant cells. With +# max_types=0 every value goes to SharedVariant as a binary descriptor plus one +# serializeBinary value, where Tuple() is zero bytes. +zero_width_dynamic_query=$( + cat <<'SQL' +SELECT + CAST(arrayMap(x -> tuple(), range(79)), 'Dynamic(max_types=0)') AS v, + CAST([arrayMap(x -> tuple(), range(13)), []], 'Dynamic(max_types=0)') AS w +FORMAT Native +SQL +) + capture() { local name="$1" local query="$2" @@ -569,4 +581,6 @@ capture "json_string_rev0.native" "${json_aux_query}" \ capture "json_flattened_rev0.native" "${json_aux_query}" \ --data-urlencode "output_format_native_use_flattened_dynamic_and_json_serialization=1" +capture "zero_width_dynamic_rev0.native" "${zero_width_dynamic_query}" + echo "Wrote fixtures to ${fixture_dir}" diff --git a/src/native/binary_value.rs b/src/native/binary_value.rs index 62a9be0..21802a4 100644 --- a/src/native/binary_value.rs +++ b/src/native/binary_value.rs @@ -56,6 +56,18 @@ const MAX_ZERO_WIDTH_ELEMENTS: usize = 1_000_000; /// or FixedString allocations. const MAX_NULL_DEFAULT_BYTES: usize = 1 << 24; +#[cfg(test)] +thread_local! { + static VISITS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Count one per-value step, so tests can bound work without timing. +#[inline] +fn count_visit() { + #[cfg(test)] + VISITS.with(|visits| visits.set(visits.get() + 1)); +} + struct NullDefaultBudget { remaining: usize, } @@ -274,12 +286,16 @@ enum ValueBuilder { offsets: Vec, elements: Box, zero_width: bool, + /// Elements are zero-width tuples, appended without a per-element loop. + fill: bool, }, Map { offsets: Vec, keys: Box, values: Box, zero_width: bool, + /// Keys and values are zero-width tuples, appended without a loop. + fill: bool, }, Tuple { fields: Vec, @@ -306,6 +322,16 @@ impl ValueBuilder { ChType::Float32 => Self::Float32(Vec::new()), ChType::Float64 => Self::Float64(Vec::new()), ChType::BFloat16 => Self::BFloat16(Vec::new()), + ChType::QBit { dimension: 0, .. } => { + return Err(BinaryValueError::Invalid( + "QBit dimension must be positive".into(), + )) + } + ChType::FixedString(0) => { + return Err(BinaryValueError::Invalid( + "FixedString width must be positive".into(), + )) + } ChType::QBit { element_type, dimension, @@ -358,17 +384,25 @@ impl ValueBuilder { // A LowCardinality single value carries no dictionary framing; the // decoded column is the plain inner column. ChType::LowCardinality(inner) => Self::new(inner)?, - ChType::Array(inner) => Self::Array { - offsets: vec![0], - elements: Box::new(Self::new(inner)?), - zero_width: has_zero_width_values(inner), - }, - ChType::Map(key, value) => Self::Map { - offsets: vec![0], - keys: Box::new(Self::new(key)?), - values: Box::new(Self::new(value)?), - zero_width: has_zero_width_values(key) && has_zero_width_values(value), - }, + ChType::Array(inner) => { + let elements = Self::new(inner)?; + Self::Array { + offsets: vec![0], + fill: elements.is_zero_width(), + elements: Box::new(elements), + zero_width: has_zero_width_values(inner), + } + } + ChType::Map(key, value) => { + let (keys, values) = (Self::new(key)?, Self::new(value)?); + Self::Map { + offsets: vec![0], + fill: keys.is_zero_width() && values.is_zero_width(), + keys: Box::new(keys), + values: Box::new(values), + zero_width: has_zero_width_values(key) && has_zero_width_values(value), + } + } ChType::Tuple(elements) => Self::Tuple { fields: elements .iter() @@ -403,12 +437,39 @@ impl ValueBuilder { } } + /// Whether values of this builder are zero bytes: `Tuple()` or a Tuple of + /// only such builders. + fn is_zero_width(&self) -> bool { + matches!(self, ValueBuilder::Tuple { fields, .. } if fields.iter().all(Self::is_zero_width)) + } + + /// Append `n` values of a zero-width builder in one step per tuple node. + fn append_zero_width(&mut self, n: usize) -> Result<(), BinaryValueError> { + if n == 0 { + return Ok(()); + } + count_visit(); + let ValueBuilder::Tuple { fields, rows } = self else { + return Err(BinaryValueError::Invalid( + "internal: zero-width fill on non-Tuple".into(), + )); + }; + for field in fields.iter_mut() { + field.append_zero_width(n)?; + } + *rows = rows + .checked_add(n) + .ok_or_else(|| BinaryValueError::Invalid("Tuple row count overflows usize".into()))?; + Ok(()) + } + /// Append one value from the reader. fn append_value( &mut self, reader: &mut ByteReader, null_budget: &mut NullDefaultBudget, ) -> Result<(), BinaryValueError> { + count_visit(); match self { // The server cannot serialize a Nothing value; only null rows // (which take the default path) are representable. @@ -484,10 +545,15 @@ impl ValueBuilder { offsets, elements, zero_width, + fill, } => { let count = read_element_count(reader, *zero_width, "Array element")?; - for _ in 0..count { - elements.append_value(reader, null_budget)?; + if *fill { + elements.append_zero_width(count)?; + } else { + for _ in 0..count { + elements.append_value(reader, null_budget)?; + } } push_end_offset(offsets, count)?; } @@ -496,11 +562,17 @@ impl ValueBuilder { keys, values, zero_width, + fill, } => { let count = read_element_count(reader, *zero_width, "Map entry")?; - for _ in 0..count { - keys.append_value(reader, null_budget)?; - values.append_value(reader, null_budget)?; + if *fill { + keys.append_zero_width(count)?; + values.append_zero_width(count)?; + } else { + for _ in 0..count { + keys.append_value(reader, null_budget)?; + values.append_value(reader, null_budget)?; + } } push_end_offset(offsets, count)?; } @@ -520,6 +592,7 @@ impl ValueBuilder { &mut self, null_budget: &mut NullDefaultBudget, ) -> Result<(), BinaryValueError> { + count_visit(); match self { ValueBuilder::Nothing { rows } => *rows += 1, ValueBuilder::Bool(values) => { @@ -1243,6 +1316,51 @@ fn check_zero_width_count( Ok(()) } +/// Rows below the returned bound hold valid zero-width values (`Tuple()` or a +/// Tuple of only such types), which encode to no bytes, so the per-row loop can +/// start there. `None` when the type is not zero-width, or a column shape, tuple +/// validity bitmap, or LowCardinality dictionary needs per-row checks. Errors +/// always come from the per-row path. +fn zero_width_bound(ch_type: &ChType, column: &Column, end: usize) -> Option { + count_visit(); + match ch_type { + ChType::SimpleAggregateFunction { inner, .. } => zero_width_bound(inner, column, end), + ChType::LowCardinality(_) if matches!(column, Column::Dictionary(_)) => None, + ChType::LowCardinality(inner) => zero_width_bound(inner, column, end), + ChType::Tuple(elements) => match column { + Column::Tuple(c) => zero_width_tuple_bound(elements.iter().map(|(_, ty)| ty), c, end), + _ => None, + }, + _ => None, + } +} + +/// [`zero_width_bound`] for one `Tuple` column under `elements`, shared with +/// the `Nested` alias. +fn zero_width_tuple_bound<'a>( + elements: impl ExactSizeIterator, + c: &TupleColumn, + end: usize, +) -> Option { + if elements.len() != c.fields.len() || c.validity.is_some() { + return None; + } + let mut bound = c.len.min(end); + for (element, field) in elements.zip(&c.fields) { + bound = bound.min(zero_width_bound(element, field, end)?); + } + Some(bound) +} + +/// First row of `start..end` the per-row loop must visit. +#[inline] +fn first_checked_row(start: usize, end: usize, bound: impl FnOnce() -> Option) -> usize { + if start == end { + return end; + } + bound().map_or(start, |bound| bound.max(start)) +} + fn fixed_slice(data: &[u8], width: usize, row: usize) -> Result<&[u8], BinaryValueError> { row.checked_mul(width) .and_then(|start| data.get(start..start.checked_add(width)?)) @@ -1281,6 +1399,7 @@ fn encode_value( column: &Column, row: usize, ) -> Result<(), BinaryValueError> { + count_visit(); macro_rules! le { ($c:expr) => {{ reject_null($c.validity.as_ref(), row)?; @@ -1474,7 +1593,8 @@ fn encode_value( let (start, end) = offset_range(&c.offsets, row)?; check_zero_width_count(end - start, || needs_element_count_cap(inner, true))?; write_varint(buf, (end - start) as u64); - for i in start..end { + let from = first_checked_row(start, end, || zero_width_bound(inner, &c.values, end)); + for i in from..end { encode_value(buf, inner, &c.values, i)?; } } @@ -1489,8 +1609,12 @@ fn encode_value( .all(|(_, ty)| needs_element_count_cap(ty, true)) })?; write_varint(buf, (end - start) as u64); - for i in start..end { - encode_tuple(buf, ch_type, fields.iter().map(|(_, ty)| ty), tuple, i)?; + let elements = || fields.iter().map(|(_, ty)| ty); + let from = first_checked_row(start, end, || { + zero_width_tuple_bound(elements(), tuple, end) + }); + for i in from..end { + encode_tuple(buf, ch_type, elements(), tuple, i)?; } } (ChType::Map(key, value), Column::Map(c)) => { @@ -1505,7 +1629,11 @@ fn encode_value( needs_element_count_cap(key, true) && needs_element_count_cap(value, true) })?; write_varint(buf, (end - start) as u64); - for i in start..end { + let from = first_checked_row(start, end, || { + let key = zero_width_bound(key, keys, end)?; + Some(key.min(zero_width_bound(value, values, end)?)) + }); + for i in from..end { encode_value(buf, key, keys, i)?; encode_value(buf, value, values, i)?; } @@ -1957,6 +2085,301 @@ mod tests { assert_eq!(c.offsets, vec![0, 5]); } + /// Run `f` and return its result with the number of per-value steps taken. + fn with_visits(f: impl FnOnce() -> T) -> (T, usize) { + VISITS.with(|visits| visits.set(0)); + let out = f(); + (out, VISITS.with(|visits| visits.get())) + } + + /// `depth` single-field Tuples around `Tuple()`. + fn nested_empty_tuple(depth: usize) -> String { + format!("{}Tuple(){}", "Tuple(".repeat(depth), ")".repeat(depth)) + } + + #[test] + fn zero_width_runs_take_constant_steps() { + let max = MAX_ZERO_WIDTH_ELEMENTS as u64; + let mut nested_arrays = varint(100); + for _ in 0..100 { + nested_arrays.extend(varint(max)); + } + let cases = [ + (format!("Array({})", nested_empty_tuple(97)), varint(max)), + ( + format!( + "Map({}, {})", + nested_empty_tuple(89), + nested_empty_tuple(89) + ), + varint(max), + ), + ("Array(Array(Tuple()))".to_string(), nested_arrays), + ( + "Array(SimpleAggregateFunction(any, SimpleAggregateFunction(any, Tuple(Tuple()))))" + .to_string(), + varint(max), + ), + ( + "Nested(a Tuple(), b Tuple(Tuple()))".to_string(), + varint(max), + ), + ]; + for (name, bytes) in cases { + let ch_type = parse(&name); + let (column, decode_steps) = with_visits(|| decode_binary_value(&ch_type, &bytes)); + let column = column.unwrap_or_else(|e| panic!("decode {name}: {e}")); + let (encoded, encode_steps) = with_visits(|| encode_one(&ch_type, &column, 0)); + assert_eq!(encoded, bytes, "{name}"); + assert!(decode_steps < 1_000, "{name}: {decode_steps} decode steps"); + assert!(encode_steps < 1_000, "{name}: {encode_steps} encode steps"); + } + + let ch_type = parse(&format!("Array({})", nested_empty_tuple(97))); + let mut column = &decode_binary_value(&ch_type, &varint(max)).unwrap(); + let Column::Array(array) = column else { + panic!("expected Array") + }; + column = &array.values; + for _ in 0..=97 { + let Column::Tuple(tuple) = column else { + panic!("expected Tuple") + }; + assert_eq!(tuple.len, MAX_ZERO_WIDTH_ELEMENTS); + column = tuple.fields.first().unwrap_or(column); + } + } + + #[test] + fn empty_zero_width_runs_take_constant_steps() { + let n = 10_000; + let wide = format!("Tuple({})", vec!["Tuple()"; 990].join(", ")); + for name in [ + format!("Array(Array({}))", nested_empty_tuple(97)), + format!( + "Array(Map({}, {}))", + nested_empty_tuple(97), + nested_empty_tuple(97) + ), + format!("Array(Array({wide}))"), + ] { + let ch_type = parse(&name); + let bytes = [varint(n as u64), vec![0; n]].concat(); + let (column, decode_steps) = with_visits(|| decode_binary_value(&ch_type, &bytes)); + let column = column.unwrap(); + let (encoded, encode_steps) = with_visits(|| encode_one(&ch_type, &column, 0)); + assert_eq!(encoded, bytes); + assert!(decode_steps < 2 * n, "{decode_steps} decode steps"); + assert!(encode_steps < 2 * n, "{encode_steps} encode steps"); + } + } + + #[test] + fn encode_zero_width_errors_match_the_per_row_path() { + fn tuple(len: usize, fields: Vec) -> Column { + Column::Tuple(TupleColumn::new(fields, len)) + } + fn array(offsets: Vec, values: Column) -> Column { + Column::Array(ArrayColumn::new(offsets, values)) + } + fn map(offsets: Vec, keys: Column, values: Column, len: usize) -> Column { + Column::Map(MapColumn::new(offsets, tuple(len, vec![keys, values]))) + } + fn dict(indices: Vec) -> Column { + Column::Dictionary(crate::column::DictionaryColumn::new( + indices, + tuple(1, vec![]), + )) + } + let nothing = || Column::Nothing(NothingColumn::new(3)); + let null_at_3 = Column::Tuple(TupleColumn::new_nullable( + vec![], + 5, + Bitmap::from_ch_null_map(&[0, 0, 0, 1, 0]), + )); + let cases = [ + ( + "Array(Tuple())", + array(vec![2, 6], tuple(4, vec![])), + "row 4 is out of range for 4 rows", + ), + ( + "Map(Tuple(), Tuple())", + map(vec![0, 5], tuple(2, vec![]), tuple(5, vec![]), 5), + "row 2 is out of range for 2 rows", + ), + ( + "Map(Tuple(), Tuple())", + map(vec![0, 5], tuple(5, vec![]), tuple(3, vec![]), 5), + "row 3 is out of range for 3 rows", + ), + ( + "Map(Tuple(), Tuple(Tuple()))", + map( + vec![0, 5], + tuple(5, vec![]), + tuple(5, vec![tuple(3, vec![])]), + 5, + ), + "row 3 is out of range for 3 rows", + ), + ( + "Nested(a Tuple())", + array(vec![0, 5], tuple(2, vec![tuple(5, vec![])])), + "row 2 is out of range for 2 rows", + ), + ( + "Nested(a Tuple())", + array(vec![0, 5], tuple(5, vec![tuple(3, vec![])])), + "row 3 is out of range for 3 rows", + ), + ( + "Array(Tuple(Tuple()))", + array(vec![0, 5], tuple(5, vec![tuple(1, vec![])])), + "row 1 is out of range for 1 rows", + ), + ( + "Array(Tuple(Tuple()))", + array(vec![0, 5], tuple(5, vec![null_at_3])), + "null row under a non-nullable type", + ), + ( + "Array(LowCardinality(Tuple()))", + array(vec![0, 3], dict(vec![0, 7, 0])), + "row 7 is out of range for 1 rows", + ), + ( + "Array(LowCardinality(Tuple()))", + array(vec![0, 3], dict(vec![0, -1, 0])), + "row 1 has negative dictionary index", + ), + ( + "Array(Tuple(Nothing))", + array(vec![0, 3], tuple(1, vec![nothing()])), + "unsupported binary value type: Nothing", + ), + ( + "Array(Tuple(Tuple(), Nothing))", + array(vec![0, 3], tuple(1, vec![tuple(3, vec![]), nothing()])), + "unsupported binary value type: Nothing", + ), + ( + "Map(Tuple(), Tuple(Nothing))", + map(vec![0, 3], tuple(1, vec![]), tuple(3, vec![nothing()]), 3), + "unsupported binary value type: Nothing", + ), + ]; + for (name, column, expected) in cases { + let mut buf = vec![0xaa]; + let err = encode_binary_value(&mut buf, &parse(name), &column, 0) + .expect_err(name) + .to_string(); + assert!(err.contains(expected), "{name}: {err}"); + assert_eq!(buf, [0xaa], "{name}"); + } + + // The dictionary fallback still encodes valid indexes. + let ch_type = parse("Array(LowCardinality(Tuple()))"); + assert_eq!( + encode_one(&ch_type, &array(vec![0, 3], dict(vec![0, 0, 0])), 0), + [3] + ); + } + + #[test] + fn byte_backed_zero_width_values_keep_their_limits() { + let n = MAX_ZERO_WIDTH_ELEMENTS + 1; + let with_count = |count: usize, tail: Vec| [varint(count as u64), tail].concat(); + let flags: Vec = (0..n).map(|i| (i % 2) as u8).collect(); + // (type, value, whether encode accepts it back) + let cases = [ + ("Array(Nullable(Nothing))", with_count(n, vec![1; n]), true), + // Encode caps Nullable(Tuple()) runs at the zero-width limit. + ("Array(Nullable(Tuple()))", with_count(n, flags), false), + ("Map(UInt32, Tuple())", with_count(n, vec![13; 4 * n]), true), + ( + "Array(Tuple(UInt8, Tuple()))", + with_count(n, vec![79; n]), + true, + ), + ( + "Array(Tuple(Tuple(), Tuple()))", + with_count(n - 1, vec![]), + true, + ), + ("Map(Tuple(), Tuple())", with_count(n - 1, vec![]), true), + ( + "Tuple(Array(Tuple()), Array(Tuple()))", + [varint(999_999), varint(999_999)].concat(), + true, + ), + ]; + for (name, bytes, reencodes) in cases { + let ch_type = parse(name); + let column = decode_binary_value(&ch_type, &bytes) + .unwrap_or_else(|e| panic!("decode {name}: {e}")); + if reencodes { + assert_eq!(encode_one(&ch_type, &column, 0), bytes, "{name}"); + } + } + // The per-container cap is unchanged for zero-width runs. + for name in ["Array(Tuple(Tuple(), Tuple()))", "Map(Tuple(), Tuple())"] { + assert!(matches!( + decode_binary_value(&parse(name), &varint(n as u64)), + Err(BinaryValueError::Invalid(_)) + )); + } + } + + #[test] + fn rejects_zero_width_fixed_string_and_qbit() { + let fixed = ChType::FixedString(0); + let qbit = ChType::QBit { + element_type: QBitElementType::Float32, + dimension: 0, + }; + let nested = ChType::Tuple(vec![ + ( + None, + ChType::Array(Box::new(ChType::Array(Box::new(fixed.clone())))), + ), + (None, ChType::String), + ]); + for ch_type in [fixed, qbit, nested] { + assert!(matches!( + decode_binary_value(&ch_type, &[0]), + Err(BinaryValueError::Invalid(ref reason)) if reason.contains("must be positive") + )); + } + } + + #[test] + fn encode_zero_width_runs_validate_child_lengths() { + let ch_type = parse("Array(Tuple())"); + let array = |offsets: Vec, tuple: TupleColumn| { + Column::Array(ArrayColumn::new(offsets, Column::Tuple(tuple))) + }; + let short = array(vec![0, 5], TupleColumn::new(vec![], 3)); + let mut buf = vec![0xaa]; + assert!(matches!( + encode_binary_value(&mut buf, &ch_type, &short, 0), + Err(BinaryValueError::Invalid(_)) + )); + assert_eq!(buf, [0xaa]); + // An empty run is not checked against the child, as before. + let empty = array(vec![7, 7], TupleColumn::new(vec![], 3)); + assert_eq!(encode_one(&ch_type, &empty, 0), [0]); + + // A tuple validity bitmap takes the per-row path. + let valid = TupleColumn::new_nullable(vec![], 3, Bitmap::all_valid(3)); + assert_eq!(encode_one(&ch_type, &array(vec![0, 3], valid), 0), [3]); + let null = TupleColumn::new_nullable(vec![], 3, Bitmap::from_ch_null_map(&[0, 1, 0])); + assert!(matches!( + encode_binary_value(&mut Vec::new(), &ch_type, &array(vec![0, 3], null), 0), + Err(BinaryValueError::Invalid(_)) + )); + } + #[test] fn reports_undecodable_types_as_unsupported() { assert!(matches!( diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 73a11d1..4ded050 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -20,6 +20,7 @@ cross-checked against the matching `.server-ref` tag `v26.8.1.2041-lts`. | `multi_block_rev0.native` | HTTP `FORMAT Native`, `max_block_size=2` | `0` | `01 02 01 6e 05 49 6e 74 33 32 0d 00 00 00 0e 00` | | `json_string_rev0.native` | HTTP `FORMAT Native`, `output_format_native_write_json_as_string=1` | `0` | `01 04 01 6a 26 4a 53 4f 4e 28 6d 61 78 5f 64 79` | | `json_flattened_rev0.native` | HTTP `FORMAT Native`, `output_format_native_use_flattened_dynamic_and_json_serialization=1` | `0` | `01 04 01 6a 26 4a 53 4f 4e 28 6d 61 78 5f 64 79` | +| `zero_width_dynamic_rev0.native` | HTTP `FORMAT Native` | `0` | `02 01 01 76 14 44 79 6e 61 6d 69 63 28 6d 61 78` | The revision-54485 fixture starts with the 10-byte `BlockInfo` preamble: `01 00 02 ff ff ff ff 03 00 00`. @@ -88,6 +89,13 @@ Decimal, and Enum promotion paths, nullable sum's flag plus conditional accumulator, and the strict one-zero-byte nothingNull state against the real server. +`zero_width_dynamic_rev0.native` has one row and two `Dynamic(max_types=0)` +columns, so every value is a SharedVariant cell. `v` holds `Array(Tuple())` +with 79 elements, cell bytes `1e 1f 00 4f`. `w` holds `Array(Array(Tuple()))` +with inner lengths 13 and 0, cell bytes `1e 1e 1f 00 02 0d 00`. `Tuple()` has +no value bytes in this format, so these counts are not backed by payload bytes. +The test checks the single-value helpers still decode and re-encode them. + Zero-row HTTP and `clickhouse-client` Native queries on the current local server emit an empty response, not a schema-bearing zero-row Native block. The decoder's zero-row block behavior remains covered by unit tests in diff --git a/tests/fixtures/zero_width_dynamic_rev0.native b/tests/fixtures/zero_width_dynamic_rev0.native new file mode 100644 index 0000000..832b1c5 Binary files /dev/null and b/tests/fixtures/zero_width_dynamic_rev0.native differ diff --git a/tests/integration.rs b/tests/integration.rs index 83b483c..2bc2c03 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -2623,3 +2623,57 @@ fn server_shared_cells_reencode_byte_for_byte() { } } } + +/// Server-written zero-width SharedVariant cells: `Array(Tuple())` counts have +/// no payload bytes behind them but are legal and must round-trip. +#[test] +fn server_zero_width_shared_cells_decode_and_reencode() { + use ch_core_rs::native::binary_value::{ + decode_binary_value, encode_binary_value, read_binary_type_prefix, write_binary_type_prefix, + }; + + let bytes = include_bytes!("fixtures/zero_width_dynamic_rev0.native"); + let batch = decode_all_bytes(bytes, &DecodeOptions::default()).expect("fixture decodes"); + let mut cells = Vec::new(); + for chunk in &batch.chunks { + for column in &chunk.columns { + collect_shared_cells(column, &mut cells); + } + } + assert_eq!( + cells, + [ + vec![0x1e, 0x1f, 0x00, 0x4f], + vec![0x1e, 0x1e, 0x1f, 0x00, 0x02, 0x0d, 0x00] + ] + ); + + // (type, offsets per Array level from the outside in, empty tuple count) + let expected: [(&str, &[&[i64]], usize); 2] = [ + ("Array(Tuple())", &[&[0, 79]], 79), + ("Array(Array(Tuple()))", &[&[0, 2], &[0, 13, 13]], 13), + ]; + for (cell, (type_name, levels, tuples)) in cells.iter().zip(expected) { + let (ch_type, consumed) = read_binary_type_prefix(cell).expect("descriptor parses"); + assert_eq!(ch_type.to_string(), type_name); + let value = decode_binary_value(&ch_type, &cell[consumed..]).expect("value decodes"); + let mut leaf = &value; + for offsets in levels { + let Column::Array(array) = leaf else { + panic!("expected Array, got {leaf:?}"); + }; + assert_eq!(array.offsets, *offsets); + leaf = &array.values; + } + let Column::Tuple(tuple) = leaf else { + panic!("expected Tuple, got {leaf:?}"); + }; + assert!(tuple.fields.is_empty()); + assert_eq!(tuple.len, tuples); + + let mut again = Vec::new(); + write_binary_type_prefix(&mut again, &ch_type).expect("descriptor encodes"); + encode_binary_value(&mut again, &ch_type, &value, 0).expect("value encodes"); + assert_eq!(&again, cell); + } +}