diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc07a5..c696a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ version; patch releases remain backward compatible. ## Unreleased +- Fix `encode_block` rejecting `SimpleAggregateFunction(f, Nullable(T))` inside + Array, Tuple, Map, Nested, and Variant with a mismatched-column error. Alias + chains below Nullable now encode and decode in any position. Type headers and + nullability checks are unchanged. - ZSTD compression frames (method byte 0x90) now decode. Before, they returned `CompressionError::UnsupportedCodec`. A ZSTD frame that claims more output than its payload can hold fails early with `InvalidUncompressedSize`. Real diff --git a/src/native/decode/mod.rs b/src/native/decode/mod.rs index 61ec3bf..2b9422c 100644 --- a/src/native/decode/mod.rs +++ b/src/native/decode/mod.rs @@ -1691,11 +1691,8 @@ fn decode_values( None }; - // A geo alias legal directly inside `Nullable` is only `Nullable(Point)` - // (the array-based kinds and `Nested` are rejected by the parser); expand it - // to its `Tuple` so the Tuple arm below handles the body after the null map, - // the ordinary `Nullable(Tuple(...))` framing. - let delegate = inner.physical_delegate_ref(); + // Resolve aliases below Nullable before dispatching the physical body. + let delegate = inner.resolved_physical_delegate_ref(); let inner = delegate.as_deref().unwrap_or(inner); // Tuple is a container of element columns decoded as a unit (each element @@ -4307,13 +4304,9 @@ fn is_resumable(ch_type: &ChType) -> bool { } } -/// Whether `Nullable(inner)` walks the same bytes in the task walker as in -/// [`skip_values`]. That walker unwraps the null map, expands at most ONE alias -/// level, then accepts only a Tuple or a leaf body; anything else (a nested -/// wrapper or container, a chained alias) is an `UnsupportedType` there, so it -/// must stay on that path here too rather than be silently accepted. +/// Whether the task walker supports the resolved physical body below Nullable. fn is_resumable_nullable_inner(inner: &ChType) -> bool { - let delegate = inner.physical_delegate_ref(); + let delegate = inner.resolved_physical_delegate_ref(); let inner = delegate.as_deref().unwrap_or(inner); match inner { ChType::Tuple(elements) => elements.iter().all(|(_, element)| is_resumable(element)), @@ -5941,9 +5934,8 @@ fn skip_values( other => other, }; - // Expand a geo alias legal directly inside `Nullable` (only `Nullable(Point)` - // -> `Tuple`), the scan-side mirror of `decode_values`. - let delegate = inner.physical_delegate_ref(); + // Resolve aliases below Nullable, matching `decode_values`. + let delegate = inner.resolved_physical_delegate_ref(); let inner = delegate.as_deref().unwrap_or(inner); // Tuple after the Nullable unwrap, mirroring `decode_values`: a diff --git a/src/native/encode/mod.rs b/src/native/encode/mod.rs index cc4d132..2e733e5 100644 --- a/src/native/encode/mod.rs +++ b/src/native/encode/mod.rs @@ -859,9 +859,8 @@ fn encode_column_values( } else { ch_type }; - // Expand a geo alias legal directly inside `Nullable` (only `Nullable(Point)` - // -> `Tuple`), so the Tuple arm below writes its body after the null map. - let delegate = value_type.physical_delegate_ref(); + // Resolve aliases below Nullable before writing the value body. + let delegate = value_type.resolved_physical_delegate_ref(); let value_type = delegate.as_deref().unwrap_or(value_type); // Tuple after the Nullable unwrap, mirroring the decode side: a // `Nullable(Tuple(...))` writes its per-row null map above, then the tuple diff --git a/src/native/encode/tests/saf_geo.rs b/src/native/encode/tests/saf_geo.rs index 888178d..e4add1d 100644 --- a/src/native/encode/tests/saf_geo.rs +++ b/src/native/encode/tests/saf_geo.rs @@ -1,4 +1,5 @@ use super::*; +use crate::native::stream_decoder::StreamDecoder; /// `SimpleAggregateFunction(sum, Float64)` over three rows; the column buffer /// is the physical Float64 (no new Column variant). @@ -827,3 +828,356 @@ fn empty_deep_array() -> Column { Column::Float64(PrimitiveColumn::new(vec![])), )) } + +fn nested_nullable_saf_batches(null_map: &[u8], wrong_variant: bool) -> Vec { + let rows = null_map.len(); + let validity = Bitmap::from_ch_null_map(null_map); + let values = if wrong_variant { + Column::UInt32(PrimitiveColumn::new_nullable(vec![13; rows], validity)) + } else { + Column::UInt64(PrimitiveColumn::new_nullable(vec![13; rows], validity)) + }; + let offsets: Vec<_> = (0..=rows as i64).collect(); + let tuple = || Column::Tuple(TupleColumn::new(vec![values.clone()], rows)); + let cases = [ + ( + "Array(SimpleAggregateFunction(max, Nullable(UInt64)))", + Column::Array(ArrayColumn::new(offsets.clone(), values.clone())), + ), + ( + "Tuple(SimpleAggregateFunction(max, Nullable(UInt64)))", + tuple(), + ), + ( + "Tuple(value SimpleAggregateFunction(max, Nullable(UInt64)))", + tuple(), + ), + ( + "Map(UInt64, SimpleAggregateFunction(max, Nullable(UInt64)))", + Column::Map(map_column( + offsets.clone(), + Column::UInt64(PrimitiveColumn::new(vec![79; rows])), + values.clone(), + )), + ), + ( + "Nested(value SimpleAggregateFunction(max, Nullable(UInt64)))", + Column::Array(ArrayColumn::new(offsets.clone(), tuple())), + ), + ( + "Array(SimpleAggregateFunction(max, SimpleAggregateFunction(max, Nullable(UInt64))))", + Column::Array(ArrayColumn::new(offsets.clone(), values.clone())), + ), + ( + "Array(Nullable(SimpleAggregateFunction(max, UInt64)))", + Column::Array(ArrayColumn::new(offsets.clone(), values.clone())), + ), + ( + "Array(SimpleAggregateFunction(max, Nullable(SimpleAggregateFunction(max, UInt64))))", + Column::Array(ArrayColumn::new(offsets.clone(), values.clone())), + ), + ( + "Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64)))", + values.clone(), + ), + ( + "SimpleAggregateFunction(max, Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))))", + values.clone(), + ), + ( + "Array(Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))))", + Column::Array(ArrayColumn::new(offsets.clone(), values.clone())), + ), + ( + "Tuple(Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))))", + tuple(), + ), + ( + "Map(UInt64, Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))))", + Column::Map(map_column( + offsets.clone(), + Column::UInt64(PrimitiveColumn::new(vec![79; rows])), + values.clone(), + )), + ), + ( + "Nested(value Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))))", + Column::Array(ArrayColumn::new(offsets.clone(), tuple())), + ), + ( + "Variant(Array(SimpleAggregateFunction(max, Nullable(UInt64))), String)", + Column::Variant(VariantColumn::try_new( + &vec![0; rows], + vec![ + Column::Array(ArrayColumn::new(offsets, values)), + Column::Utf8(utf8_column(&[])), + ], + ).unwrap()), + ), + ]; + cases + .into_iter() + .map(|(name, column)| { + ColBatch::new( + Schema::new(vec![Field { + name: "value".into(), + ch_type: parse_ch_type(name).unwrap(), + }]), + vec![column], + rows, + ) + }) + .collect() +} + +#[test] +fn roundtrip_nested_nullable_saf() { + for null_map in [&[0, 0][..], &[0, 1], &[1, 1], &[]] { + for batch in nested_nullable_saf_batches(null_map, false) { + for revision in [0, DBMS_TCP_PROTOCOL_VERSION] { + if batch.num_rows == 0 { + let bytes = encode_block( + &batch, + &EncodeOptions { + protocol_revision: revision, + ..EncodeOptions::default() + }, + ) + .unwrap(); + let decoded = decode_all_bytes( + &bytes, + &DecodeOptions { + protocol_revision: revision, + ..DecodeOptions::default() + }, + ) + .unwrap(); + assert_eq!(decoded.schema, batch.schema); + assert_eq!(decoded.num_rows(), 0); + } else { + roundtrip(&batch, revision); + } + } + } + } +} + +#[test] +fn nested_nullable_saf_rejects_wrong_buffers() { + for null_map in [&[0, 0][..], &[0, 1], &[]] { + for batch in nested_nullable_saf_batches(null_map, true) { + assert!(matches!( + encode_block(&batch, &EncodeOptions::default()), + Err(EncodeError::InconsistentBatch { .. }) + )); + } + } +} + +#[test] +fn nested_saf_preserves_nullability_validation() { + for mut batch in nested_nullable_saf_batches(&[0, 1], false) { + let name = batch.schema.fields[0].ch_type.to_string(); + let non_nullable = name + .replace("Nullable(UInt64)", "UInt64") + .replace( + "Nullable(SimpleAggregateFunction(max, UInt64))", + "SimpleAggregateFunction(max, UInt64)", + ) + .replace( + "Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64)))", + "SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64))", + ); + assert_ne!(name, non_nullable); + batch.schema.fields[0].ch_type = parse_ch_type(&non_nullable).unwrap(); + assert!(matches!( + encode_block(&batch, &EncodeOptions::default()), + Err(EncodeError::InconsistentBatch { .. }) + )); + } + + let mut batch = nested_nullable_saf_batches(&[0, 1], false).remove(0); + let Column::Array(array) = &mut batch.columns[0] else { + panic!("expected Array"); + }; + let Column::UInt64(values) = array.values.as_mut() else { + panic!("expected UInt64"); + }; + values.validity = Some(Bitmap::from_ch_null_map(&[0])); + assert!(matches!( + encode_block(&batch, &EncodeOptions::default()), + Err(EncodeError::InconsistentBatch { .. }) + )); +} + +#[test] +fn double_nullable_is_inconsistent_batch() { + let nullable = |inner| ChType::Nullable(Box::new(inner)); + let values = || { + Column::UInt64(PrimitiveColumn::new_nullable( + vec![13, 79], + Bitmap::from_ch_null_map(&[0, 1]), + )) + }; + let cases = [ + (nullable(nullable(ChType::UInt64)), values()), + ( + ChType::Array(Box::new(nullable(nullable(ChType::UInt64)))), + Column::Array(ArrayColumn::new(vec![0, 1, 2], values())), + ), + ]; + for (ch_type, column) in cases { + let batch = ColBatch::new( + Schema::new(vec![Field { + name: "value".into(), + ch_type, + }]), + vec![column], + 2, + ); + assert!(matches!( + encode_block(&batch, &EncodeOptions::default()), + Err(EncodeError::InconsistentBatch { .. }) + )); + } +} + +#[test] +fn nested_saf_map_key_validation() { + for rows in [0, 2] { + let batch = ColBatch::new( + Schema::new(vec![Field { + name: "value".into(), + ch_type: parse_ch_type( + "Map(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64)), UInt64)", + ) + .unwrap(), + }]), + vec![Column::Map(map_column( + (0..=rows as i64).collect(), + Column::UInt64(PrimitiveColumn::new(vec![13; rows])), + Column::UInt64(PrimitiveColumn::new(vec![79; rows])), + ))], + rows, + ); + if rows > 0 { + roundtrip(&batch, 0); + roundtrip(&batch, DBMS_TCP_PROTOCOL_VERSION); + } else { + encode_block(&batch, &EncodeOptions::default()).unwrap(); + } + let mut invalid = batch.clone(); + let Column::Map(map) = &mut invalid.columns[0] else { + panic!("expected Map"); + }; + let Column::Tuple(entries) = map.entries.as_mut() else { + panic!("expected Tuple"); + }; + entries.fields[0] = Column::UInt32(PrimitiveColumn::new(vec![13; rows])); + assert!(matches!( + encode_block(&invalid, &EncodeOptions::default()), + Err(EncodeError::InconsistentBatch { .. }) + )); + let mut invalid = batch; + invalid.schema.fields[0].ch_type = ChType::Map( + Box::new(ChType::SimpleAggregateFunction { + func: "max".into(), + inner: Box::new(ChType::Nullable(Box::new(ChType::UInt64))), + }), + Box::new(ChType::UInt64), + ); + assert!(matches!( + encode_block(&invalid, &EncodeOptions::default()), + Err(EncodeError::UnsupportedType { .. }) + )); + } +} + +#[test] +fn rev0_frames_nullable_chained_saf_bytes() { + let name = "Array(SimpleAggregateFunction(max, Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, UInt64)))))"; + for null_map in [&[0, 0][..], &[0, 1], &[1, 1], &[]] { + let rows = null_map.len(); + let batch = ColBatch::new( + Schema::new(vec![Field { + name: "value".into(), + ch_type: parse_ch_type(name).unwrap(), + }]), + vec![Column::Array(ArrayColumn::new( + (0..=rows as i64).collect(), + Column::UInt64(PrimitiveColumn::new_nullable( + vec![13; rows], + Bitmap::from_ch_null_map(null_map), + )), + ))], + rows, + ); + let mut expected = vec![1, rows as u8]; + write_string(&mut expected, b"value"); + write_string(&mut expected, name.as_bytes()); + for offset in 1..=rows as u64 { + expected.extend_from_slice(&offset.to_le_bytes()); + } + expected.extend_from_slice(null_map); + for _ in 0..rows { + expected.extend_from_slice(&13u64.to_le_bytes()); + } + assert_eq!( + encode_block(&batch, &EncodeOptions::default()).unwrap(), + expected + ); + if rows > 0 { + roundtrip(&batch, 0); + roundtrip(&batch, DBMS_TCP_PROTOCOL_VERSION); + } + } +} + +fn roundtrip_streamed_saf(batch: &ColBatch, revision: u64) { + let bytes = encode_block( + batch, + &EncodeOptions { + protocol_revision: revision, + ..EncodeOptions::default() + }, + ) + .unwrap(); + let mut decoder = StreamDecoder::new(DecodeOptions { + protocol_revision: revision, + ..DecodeOptions::default() + }); + let mut chunks = Vec::new(); + for byte in bytes.chunks(1) { + chunks.extend(decoder.feed(byte).unwrap()); + } + chunks.extend(decoder.finish().unwrap()); + assert_eq!(chunks.len(), 1); + assert_batches_eq(batch, &chunks[0]); +} + +#[test] +fn stream_nested_nullable_saf() { + for null_map in [&[0, 0][..], &[0, 1], &[1, 1]] { + for batch in nested_nullable_saf_batches(null_map, false) { + for revision in [0, DBMS_TCP_PROTOCOL_VERSION] { + roundtrip_streamed_saf(&batch, revision); + } + } + } +} + +#[test] +fn roundtrip_nullable_saf_point() { + for name in [ + "Nullable(SimpleAggregateFunction(max, Point))", + "Nullable(SimpleAggregateFunction(max, SimpleAggregateFunction(max, Point)))", + "SimpleAggregateFunction(max, Nullable(SimpleAggregateFunction(max, Point)))", + ] { + let mut batch = nullable_point_batch(); + batch.schema.fields[0].ch_type = parse_ch_type(name).unwrap(); + for revision in [0, DBMS_TCP_PROTOCOL_VERSION] { + roundtrip(&batch, revision); + roundtrip_streamed_saf(&batch, revision); + } + } +} diff --git a/src/native/encode/validate.rs b/src/native/encode/validate.rs index e9e450b..2c60d1a 100644 --- a/src/native/encode/validate.rs +++ b/src/native/encode/validate.rs @@ -1830,8 +1830,8 @@ fn validate_json<'a>( Ok(()) } -/// Whether `value_type` (the unwrapped inner value type) and `column` form a -/// supported, matching pair this encoder can write. +/// Whether `value_type` and `column` form a supported, matching pair this +/// encoder can write. /// /// This must list exactly the matching arms of [`encode_column_body`], plus /// `LowCardinality`'s [`encode_low_cardinality_data`] path and `Array`'s @@ -1840,6 +1840,9 @@ fn validate_json<'a>( /// column before any bytes are written, which is what lets [`write_block_into`] /// treat the body match as structurally infallible. fn column_variant_matches(value_type: &ChType, column: &Column) -> bool { + if let ChType::Nullable(inner) = value_type { + return column_variant_matches(inner, column); + } // A name-decoration alias (SimpleAggregateFunction, geo, Nested) matches the // column its physical delegate would, since decode produces the delegate's // Column variant (no new variant). Expand and recurse before the pair checks @@ -1861,35 +1864,28 @@ fn column_variant_matches(value_type: &ChType, column: &Column) -> bool { } // `Array(T)` matches only if the flattened element column matches the // element value type in turn, recursing the same way `decode_array` decodes - // through `decode_values`. A `Nullable` element unwraps to its inner here - // (the element column carries the inner variant plus validity), exactly like - // the top-level unwrap in `validate_column`. + // through `decode_values`. if let (ChType::Array(inner), Column::Array(c)) = (value_type, column) { - return column_variant_matches(inner.inner(), c.values.as_ref()); + return column_variant_matches(inner, c.values.as_ref()); } // `Tuple(T1, ...)` matches only if the buffer carries exactly one field // column per declared element and each matches its element value type in - // turn (a `Nullable` element unwraps to its inner, like the Array arm - // above; a `LowCardinality` or nested container element recurses through - // this function's own dispatch). The zero-element `Tuple()` matches a - // zero-field `TupleColumn`. + // turn. The zero-element `Tuple()` matches a zero-field `TupleColumn`. if let (ChType::Tuple(elements), Column::Tuple(c)) = (value_type, column) { return elements.len() == c.fields.len() && elements .iter() .zip(&c.fields) - .all(|((_, t), col)| column_variant_matches(t.inner(), col)); + .all(|((_, t), col)| column_variant_matches(t, col)); } // `Map(K, V)` matches only if the entries buffer is a two-field Tuple - // column whose fields match the key and value types in turn (a `Nullable` - // value unwraps to its inner like everywhere else; a legal key is never - // `Nullable`, so its `inner()` is a no-op). + // column whose fields match the key and value types in turn. if let (ChType::Map(key, value), Column::Map(c)) = (value_type, column) { return match c.entries.as_ref() { Column::Tuple(t) => { t.fields.len() == 2 - && column_variant_matches(key.inner(), &t.fields[0]) - && column_variant_matches(value.inner(), &t.fields[1]) + && column_variant_matches(key, &t.fields[0]) + && column_variant_matches(value, &t.fields[1]) } _ => false, }; @@ -1899,7 +1895,7 @@ fn column_variant_matches(value_type: &ChType, column: &Column) -> bool { && alternatives .iter() .zip(&c.variants) - .all(|(t, child)| column_variant_matches(t.inner(), child)); + .all(|(t, child)| column_variant_matches(t, child)); } if let (ChType::Dynamic { .. }, Column::Dynamic(_)) = (value_type, column) { return true; diff --git a/src/native/stream_decoder.rs b/src/native/stream_decoder.rs index 46d069a..e625179 100644 --- a/src/native/stream_decoder.rs +++ b/src/native/stream_decoder.rs @@ -1441,10 +1441,18 @@ mod tests { scanner.scan(&data[..header_end + 1]).unwrap(), BlockBoundary::NeedMore ); - let scanned = scanner.scan(&data).unwrap_err(); - let decoded = block_end(&data, &DecodeOptions::default()).unwrap_err(); - assert!(matches!(scanned, DecodeError::UnsupportedType { .. })); - assert_eq!(scanned.to_string(), decoded.to_string()); + assert_eq!( + scanner.scan(&data).unwrap(), + BlockBoundary::Complete { + consumed: data.len(), + num_columns: 1, + num_rows: 2, + } + ); + assert_eq!( + block_end(&data, &DecodeOptions::default()).unwrap(), + Some(data.len()) + ); } #[test]