From ff4125cfac971193c12e490b113d1b5b774ef36e Mon Sep 17 00:00:00 2001 From: Joe S Date: Tue, 17 Nov 2026 15:46:26 -0800 Subject: [PATCH 1/2] Decode JSON values inside Dynamic shared cells decode_binary_value and encode_binary_value now handle JSON, including JSON nested in Array, Tuple, Map and Nullable. Before, they returned Unsupported and bindings fell back to raw bytes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + CODEC_CONTRACT.md | 93 +- COMPLETENESS.md | 10 +- INTEGRATING.md | 7 + README.md | 5 + scripts/gen_fixtures.sh | 55 + src/native/binary_value.rs | 1375 ++++++++++++++++- src/native/type_binary.rs | 48 +- tests/fixtures/README.md | 44 + tests/fixtures/json_mixed_rows_rev0.native | Bin 0 -> 171 bytes tests/fixtures/json_multipart_rev0.native | Bin 0 -> 128 bytes tests/fixtures/json_multipart_rev54460.native | Bin 0 -> 148 bytes tests/fixtures/json_multipart_rev54485.native | Bin 0 -> 150 bytes tests/fixtures/json_shared_rev0.native | Bin 0 -> 1648 bytes tests/fixtures/json_shared_rev54460.native | Bin 0 -> 1665 bytes tests/fixtures/json_shared_rev54485.native | Bin 0 -> 1656 bytes tests/integration.rs | 251 +++ 17 files changed, 1819 insertions(+), 77 deletions(-) create mode 100644 tests/fixtures/json_mixed_rows_rev0.native create mode 100644 tests/fixtures/json_multipart_rev0.native create mode 100644 tests/fixtures/json_multipart_rev54460.native create mode 100644 tests/fixtures/json_multipart_rev54485.native create mode 100644 tests/fixtures/json_shared_rev0.native create mode 100644 tests/fixtures/json_shared_rev54460.native create mode 100644 tests/fixtures/json_shared_rev54485.native diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b4e894..4950009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,14 @@ version; patch releases remain backward compatible. Array, Tuple, Map, Nested, and Variant with a mismatched-column error. Alias chains below Nullable now encode and decode in any position. Type headers and nullability checks are unchanged. +- `decode_binary_value` and `encode_binary_value` now support structured JSON + instead of returning `Unsupported`, including JSON inside arrays, tuples, and + maps. These on-demand helpers preserve typed paths and tagged shared paths. + Bulk Dynamic and JSON shared cells stay opaque. Non-null Nothing values now + return `Invalid` in both helpers. +- Binary type descriptors with 1001 to 2001 nodes now decode. The 2001-node + budget is a crate resource limit. Type-depth errors now say + `nesting exceeds 100` instead of `type nesting too deep`. - ZSTD compression frames (method byte 0x90) now decode. Before, they returned `CompressionError::UnsupportedCodec`. A ZSTD frame that claims more output than its payload can hold fails early with `InvalidUncompressedSize`. Real diff --git a/CODEC_CONTRACT.md b/CODEC_CONTRACT.md index 8c5a13b..3c68f47 100644 --- a/CODEC_CONTRACT.md +++ b/CODEC_CONTRACT.md @@ -2483,8 +2483,14 @@ parse and value materialization into the bulk path. On demand, one-row column, and `write_binary_type_prefix` plus `encode_binary_value` write one row of a column back as a blob, so a caller can build the SharedVariant child or JSON shared data that `encode_block` takes. The encode side covers the -same types as the decode side and reproduces the server's value bytes. -Descriptors also match, except SimpleAggregateFunction and AggregateFunction +same types as the decode side and uses the server's single-value layout. +This includes JSON at the root or nested in supported containers. Single-value +JSON decodes into the same structured column model as bulk JSON: declared typed +paths keep their child columns, while all other paths use tagged shared-data +buffers. It does not infer a separate Dynamic child for each discovered path. +JSON path order may differ because the server iterates unordered maps. +Descriptors also match, except JSON path lists are canonicalized and +SimpleAggregateFunction and AggregateFunction use the custom-name `0x2c` form that the server accepts instead of its native aggregate descriptor forms. The server writes SimpleAggregateFunction with `0x2e` and AggregateFunction with `0x25`; AggregateFunction value encoding @@ -2551,8 +2557,19 @@ headers and Dynamic tables use `DataTypesBinaryEncoding` descriptors instead. The public `decode_*_binary_types` and `encode_*_binary_types` APIs select that out-of-band form while keeping `DecodeOptions` and `EncodeOptions` source compatible. The descriptor reader is bounded by `MAX_TYPE_DEPTH`, complexity -1000, and list count 1,000,000, and rejects illegal wrappers, reserved tags, and +2001, and list count 1,000,000, and rejects illegal wrappers, reserved tags, and unsupported descriptor forms before payload decode. +The 2001-node budget is a crate resource policy, not server parity. It covers +a JSON root with 1000 Nullable scalar typed paths, while 1000 typed +Array(Nullable(Int64)) paths require 3001 nodes and remain rejected. +ClickHouse decodes shared-cell descriptors without a complexity limit. Its +Native binary-header setting `input_format_binary_max_type_complexity` +defaults to 1000 and can be raised independently. The server's typed-path +count cap remains 1000. These server limits are confirmed at +`v26.8.1.2041-lts` in +`src/DataTypes/DataTypesBinaryEncoding.cpp::decodeDataTypeImpl`, +`src/DataTypes/DataTypeObject.h::MAX_TYPED_PATHS`, and +`src/Core/FormatFactorySettings.h`. A zero-row Native block carries only the logical column header. NativeWriter suppresses the structure word, type table, child prefixes, routing, and bodies. @@ -2695,11 +2712,79 @@ bytes; the type-aware per-column row-count guard excludes JSON (and any containe that bottoms out in it) and bounds it by its own read-before-allocate decode. SharedVariant and shared-data values are opaque binary blobs, one binary data-type descriptor followed by one `serializeBinary` payload, the same shape as -a Dynamic SharedVariant cell, kept opaque and never materialized. A zero-row +a Dynamic SharedVariant cell, kept opaque during bulk decoding. The +single-value binary helpers can materialize these cells on demand. A zero-row Native block writes only the column header (no structure word, path list, or bodies); `empty_column` builds the declared typed paths with empty children and empty shared data. +**Single-value binary payload:** JSON inside a shared cell uses a different +layout from the bulk structure above. Its type descriptor starts with `0x30`, +then UInt8 descriptor version `0`, VarUInt `max_dynamic_paths`, UInt8 +`max_dynamic_types`, a VarUInt-counted typed-path list of path strings and +recursive type descriptors, a VarUInt-counted skip-path list, and a +VarUInt-counted skip-regexp list. Strings use VarUInt byte lengths. + +The value starts with a VarUInt path count. Each entry is a path string followed +by its value. Declared typed paths use their declared serializer without another +type descriptor. Other paths carry a binary type descriptor followed by one +value. The server writes typed paths first, then non-null dynamic paths, then +shared-data paths, with no global ordering guarantee. Typed defaults are +included even when a path was absent in the input document. An empty untyped +object is one zero byte. The value has no bulk structure version word or path +table, and its layout is the same inside V1 and V2 Dynamic shared cells. + +`decode_binary_value` preserves typed children and stores other entries in +`StructuredJson` shared buffers, sorted by path within each row. Paths are +compared as raw bytes and duplicates are rejected. Untyped path bytes are +preserved without UTF-8 validation, like bulk shared paths. +`encode_binary_value` accepts structured JSON +from either this helper or bulk decoding. Typed paths require supported +single-value codecs, so a declared Variant, Dynamic, or AggregateFunction typed +path still returns `BinaryValueError::Unsupported`. Untyped paths can retain +Variant and Dynamic payloads as opaque tagged cells. When encoding a +bulk-decoded JSON column, only the selected row's runtime values need codecs. +An unrelated row's unsupported child does not prevent encoding. A selected +runtime value that reaches a Dynamic or Variant leaf still returns +`Unsupported`, including such leaves inside nonempty containers. Empty +containers do not visit their leaves and can encode. +A preexisting shared path +tagged Nothing is retained as the one-byte `0x00` cell. Normal JSON input omits +null untyped paths, but the row serializer copies raw shared entries verbatim. +Nothing has no non-null row payload. The opaque value walker permits it only +as a tagged Dynamic NULL, including inside Array(Dynamic). Arrays and maps +with a Nothing element, key, or value type must be empty. Nullable(Nothing) +must carry a null flag. +Non-null Nothing values return `BinaryValueError::Invalid` in both helpers. +Runtime child descriptors written by the JSON helper use the same 2001-node +complexity limit as the descriptor reader. +Missing typed paths and zero-width default nodes consume units from the +per-value materialization budget, even when they allocate no data bytes. +Opaque values are framed with a plan compiled once per binary descriptor. +Aliases and zero-width children are resolved during compilation, so repeated +Array and Map values do not re-walk or clone their type trees. +The helper preserves serialized paths without applying `SKIP` or `SKIP REGEXP` +again. Normal server output has already applied its input skip rules. This +differs from the server's row deserializer, which filters incoming paths using +byte-prefix SKIP matching and RE2 full-match regexps. +The bulk decoder continues to retain shared cells without parsing each one. +These per-value layouts are +**CONFIRMED** at `v26.8.1.2041-lts` in +`src/DataTypes/Serializations/SerializationObject.cpp` +(`SerializationObject::serializeBinary`, `deserializeBinary`), +`src/DataTypes/Serializations/SerializationDynamic.cpp` +(`SerializationDynamic::serializeBinary`), +`src/DataTypes/Serializations/SerializationNothing.h` +(`SerializationNothing::serializeBinary`, `deserializeBinary`), +`src/Columns/ColumnDynamic.cpp` +(`ColumnDynamic::serializeValueIntoSharedVariant`), and +`src/DataTypes/DataTypesBinaryEncoding.cpp` +(`encodeDataTypeImpl`, `decodeDataTypeImpl`). +Shared-path ordering and row skip rules are confirmed in +`src/Columns/ColumnObject.cpp::findPathLowerBoundInSharedData` and +`src/DataTypes/Serializations/SerializationObject.cpp` +(`deserializeBinary`, `shouldSkipPath`). + **Arrow export:** a structured body exports as an Arrow struct (`+s`) whose children are, in fixed order: one child per declared typed path (each the path type's own schema and buffers), one Dense Union child per dynamic path (the same diff --git a/COMPLETENESS.md b/COMPLETENESS.md index d3b39c5..4d3d3bf 100644 --- a/COMPLETENESS.md +++ b/COMPLETENESS.md @@ -64,6 +64,11 @@ default; the user may override it. types. This crate does not implement that separate layout yet, so `DBMS_TCP_PROTOCOL_VERSION` and the framed fixture intentionally remain at negotiated revision 54485. BlockInfo and Geometry framing are unchanged. +- **JSON shared-cell checkpoint, 2026-11-17:** the single-value binary codec + decodes and encodes JSON, including JSON nested in containers. Typed paths + retain their declared columns, and other paths retain tagged shared cells. + Real-server regressions at `v26.8.1.2041-lts` cover revisions 0, 54460 and + 54485. Bulk Dynamic and JSON shared buffers stay opaque. - **AggregateFunction checkpoint:** decode, encode, streaming, Arrow LargeBinary export, real-server fixtures, and live INSERT coverage are complete for exact base `count` with zero or one argument, canonical @@ -651,7 +656,10 @@ where the across-release churn lives. under the type-aware row-count guard. `Nullable(JSON)` is legal (top-level null map), `LowCardinality(JSON)` illegal, and JSON composes in Array/Tuple/Map/Variant/Dynamic and as its own typed path (nested), bounded - by `MAX_TYPE_DEPTH`. Arrow export is a struct of the typed paths, the + by `MAX_TYPE_DEPTH`. The single-value binary helpers also decode and + encode JSON inside shared cells, preserving typed children and retaining + other paths as tagged shared values. Arrow export is a struct of the + typed paths, the result-wide dynamic-path Dense Unions, and a `_shared_data` LargeList of `{paths utf8, values binary}`, with a Text body exporting as utf8; binary type descriptor tag 0x30. Legacy `Object('json')` is not registered at the diff --git a/INTEGRATING.md b/INTEGRATING.md index 1c4d0f5..8131d2c 100644 --- a/INTEGRATING.md +++ b/INTEGRATING.md @@ -324,6 +324,13 @@ The physical column buffers are Arrow-shaped: The core covers the full ClickHouse scalar set plus containers (`Array`, `Tuple`, `Map`, `Nested`), `Variant`, `Dynamic`, `JSON`, wide integers, `LowCardinality(T)`, the geo types, `QBit`, a set of `AggregateFunction`/`SimpleAggregateFunction` states, and `Nullable(T)` over the supported inner types. The "Supported types" list in `README.md` is the current enumeration, and `CODEC_CONTRACT.md` is the per-type wire and buffer contract; this guide does not repeat the list so it cannot drift. Unsupported types fail with `DecodeError::UnsupportedType` on decode (and `EncodeError::UnsupportedType` on encode). +Dynamic shared cells and JSON shared-data values retain their binary type +descriptor and single-value payload. Decode them on demand with +`native::binary_value::read_binary_type_prefix` and `decode_binary_value`. +This supports JSON at the root and inside containers. JSON typed paths retain +their declared child columns, and its other paths use tagged shared-data +buffers. Bindings can process those buffers like standalone JSON. + ClickHouse `String` is arbitrary bytes, not guaranteed UTF-8. The core stores the raw bytes. The current Arrow export uses Arrow utf8 format because the physical layout is offsets plus data, but strict Arrow consumers may reject invalid UTF-8 when importing or later validating the array. A binding that needs byte-faithful behavior should validate first, choose a bytes/binary fallback, or return a clear unsupported-for-Arrow error for invalid strings. ## Consumption Strategies diff --git a/README.md b/README.md index 6ffbe58..e1d89f4 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,11 @@ block bytes for `INSERT` with `native::encode::encode_block` or `decode_*_binary_types` entry points. Bytes from `encode_*_binary_types` require `input_format_native_decode_types_in_binary_format=1` on the receiving INSERT. +Dynamic shared cells and JSON shared-data values stay as binary blobs during +bulk decode. Use `native::binary_value::read_binary_type_prefix` and +`decode_binary_value` to decode a cell on demand, including JSON inside +containers. The corresponding writers build cells for encoding. + ## Repo layout The root package remains `ch-core-rs`. The workspace also contains diff --git a/scripts/gen_fixtures.sh b/scripts/gen_fixtures.sh index 0b82ff1..19c3e38 100755 --- a/scripts/gen_fixtures.sh +++ b/scripts/gen_fixtures.sh @@ -547,6 +547,24 @@ FORMAT Native SQL ) +json_shared_query=$( + cat <<'SQL' +WITH number AS n +SELECT + CAST(multiIf(n = 0, '{"n":{"x":13},"z":79,"a":17}', n = 1, '{}', n = 2, '{"n":{"x":null}}', '{"n":{"x":79}}')::JSON, 'Dynamic(max_types=0)') AS j, + CAST([multiIf(n = 0, '{"k":1}', n = 1, '{}', n = 2, '{"other":null}', '{"k":79}')::JSON, '{}'::JSON], 'Dynamic(max_types=0)') AS aj, + CAST(tuple('{"k":79}'::JSON, toUInt8(17)), 'Dynamic(max_types=0)') AS tj, + CAST(map('user_1', '{"k":13}'::JSON), 'Dynamic(max_types=0)') AS mj, + CAST(multiIf(n = 0, '{"a":13,"b":"user_1","d":79,"dropped":17,"omit_me":1}', n = 1, '{}', n = 2, '{"a":null,"d":null}', '{"a":79,"later":13}') AS JSON(max_dynamic_paths=0, max_dynamic_types=0, a Nullable(Int64), b String, SKIP dropped, SKIP REGEXP '^omit')) AS typed, + CAST(typed, 'Dynamic(max_types=0)') AS typed_shared, + CAST('{"mix":[1,"user_1",null]}'::JSON, 'Dynamic(max_types=0)') AS heterogeneous, + CAST('{"child":{"k":13}}' AS JSON(child JSON)) AS nested, + CAST(nested, 'Dynamic(max_types=0)') AS nested_shared +FROM numbers(4) +FORMAT Native +SQL +) + capture() { local name="$1" local query="$2" @@ -583,4 +601,41 @@ capture "json_flattened_rev0.native" "${json_aux_query}" \ capture "zero_width_dynamic_rev0.native" "${zero_width_dynamic_query}" +# JSON single-value payloads inside Dynamic SharedVariant cells. Revisions 0 +# and 54460 exercise Dynamic V1, while 54485 exercises V2. +json_mixed_rows_query=$( + cat <<'SQL' +SELECT CAST(multiIf(number = 0, '{}', number = 1, '{"mix":[1,"user_1",null]}', number = 2, '{"mix":13}', '{"mix":[]}') AS JSON) AS j +FROM numbers(4) +FORMAT Native +SQL +) +capture "json_mixed_rows_rev0.native" "${json_mixed_rows_query}" +capture "json_shared_rev0.native" "${json_shared_query}" +capture "json_shared_rev54460.native" "${json_shared_query}" \ + --data-urlencode "client_protocol_version=54460" +capture "json_shared_rev54485.native" "${json_shared_query}" \ + --data-urlencode "client_protocol_version=54485" + +json_table="ch_core_json_shared_fixture_$$" +execute() { + curl -sS --fail "${url}" --user "${user}:${password}" --data-binary "$1" +} +cleanup_json_table() { + execute "DROP TABLE IF EXISTS ${json_table}" >/dev/null +} +trap cleanup_json_table EXIT +execute "CREATE TABLE ${json_table} (id UInt8, d Dynamic) ENGINE=MergeTree ORDER BY id" +execute "SYSTEM STOP MERGES ${json_table}" +execute "INSERT INTO ${json_table} SELECT 0, '{\"n\":{\"x\":13}}'::JSON" +execute "INSERT INTO ${json_table} SELECT 1, ['{\"k\":1}'::JSON]" +json_multipart_query="SELECT id, d FROM ${json_table} ORDER BY id FORMAT Native" +capture "json_multipart_rev0.native" "${json_multipart_query}" +capture "json_multipart_rev54460.native" "${json_multipart_query}" \ + --data-urlencode "client_protocol_version=54460" +capture "json_multipart_rev54485.native" "${json_multipart_query}" \ + --data-urlencode "client_protocol_version=54485" +cleanup_json_table +trap - EXIT + echo "Wrote fixtures to ${fixture_dir}" diff --git a/src/native/binary_value.rs b/src/native/binary_value.rs index 21802a4..da4991a 100644 --- a/src/native/binary_value.rs +++ b/src/native/binary_value.rs @@ -20,10 +20,14 @@ //! - SimpleAggregateFunction/geo/Geometry/Nested expand through //! [`ChType::physical_delegate`], like every bulk dispatcher. //! -//! `Variant`, `Dynamic`, and `AggregateFunction` values (an opaque -//! per-function state) have no decoding here and report `Unsupported`, as do -//! descriptors the crate does not parse (JSON), so a caller can keep -//! those cells as raw bytes. Truncated payloads and trailing bytes after the +//! - JSON: VarUInt path count, then path strings and row values. Declared +//! typed paths use their declared serializer. Other paths carry a binary type +//! descriptor and remain opaque cells in `StructuredJson::shared_values`. +//! +//! `Variant`, `Dynamic`, and `AggregateFunction` values report `Unsupported`. +//! JSON typed paths require supported single-value codecs. Untyped path cells +//! may contain Dynamic or Variant values, which are framed without materializing +//! their contents. Truncated payloads and trailing bytes after the //! value are `Invalid`. //! //! [`write_binary_type_prefix`] and [`encode_binary_value`] are the inverses: @@ -34,11 +38,14 @@ use std::io; use crate::bitmap::Bitmap; use crate::column::{ - ArrayColumn, BoolColumn, Column, DecimalColumn, FixedBinaryColumn, MapColumn, NothingColumn, - PrimitiveColumn, QBitColumn, TupleColumn, Utf8Column, + ArrayColumn, BoolColumn, Column, DecimalColumn, DynamicChild, FixedBinaryColumn, JsonBody, + JsonColumn, MapColumn, NothingColumn, PrimitiveColumn, QBitColumn, StructuredJson, TupleColumn, + Utf8Column, }; use crate::native::protocol::MAX_TYPE_DEPTH; -use crate::native::type_binary::{read_binary_type, write_binary_type, BinaryTypeError}; +use crate::native::type_binary::{ + read_binary_type, write_binary_type, BinaryTypeError, MAX_BINARY_TYPE_COMPLEXITY, +}; use crate::native::varint::{write_varint, ByteReader}; use crate::schema::{ChType, QBitElementType, GEOMETRY_EXPANSION_DEPTH}; @@ -46,7 +53,7 @@ use crate::schema::{ChType, QBitElementType, GEOMETRY_EXPANSION_DEPTH}; /// (`Tuple()`, `Nothing`); every other count is bounded by the bytes present. const MAX_ZERO_WIDTH_ELEMENTS: usize = 1_000_000; -/// Maximum cumulative logical buffer bytes materialized for null placeholders +/// Maximum cumulative null-placeholder bytes and zero-width default work /// while decoding one single value. /// /// A Nullable single value carries only its one-byte null flag, but Arrow-shaped @@ -130,7 +137,7 @@ impl std::error::Error for BinaryValueError {} /// /// Returns the type and the number of descriptor bytes consumed, so the caller /// can hand the rest of the cell to [`decode_binary_value`]. An unparseable -/// server type (JSON, Set, Function) is `Unsupported`; a malformed +/// server type (Set, Function) is `Unsupported`; a malformed /// descriptor is `Invalid`. pub fn read_binary_type_prefix(bytes: &[u8]) -> Result<(ChType, usize), BinaryValueError> { let mut reader = ByteReader::new(bytes); @@ -147,8 +154,10 @@ pub fn read_binary_type_prefix(bytes: &[u8]) -> Result<(ChType, usize), BinaryVa /// type. `bytes` must contain the value and nothing else: truncation and /// trailing bytes are both `Invalid`. Null placeholders that have no backing /// value bytes share a cumulative 16 MiB materialization budget for this call. +/// Missing typed paths and zero-width default nodes also consume budget units. pub fn decode_binary_value(ch_type: &ChType, bytes: &[u8]) -> Result { - let mut builder = ValueBuilder::new(ch_type)?; + check_type_depth(ch_type)?; + let mut builder = ValueBuilder::new(ch_type, 0)?; let mut reader = ByteReader::new(bytes); let mut null_budget = NullDefaultBudget::new(); builder.append_value(&mut reader, &mut null_budget)?; @@ -301,12 +310,26 @@ enum ValueBuilder { fields: Vec, rows: usize, }, + Json(Box), +} + +struct JsonValueBuilder { + typed: Vec<(String, ValueBuilder)>, + offsets: Vec, + paths: Utf8Column, + values: Utf8Column, + depth: usize, + seen: Vec, + path_order: Vec<(std::ops::Range, std::ops::Range)>, } impl ValueBuilder { - fn new(ch_type: &ChType) -> Result { + fn new(ch_type: &ChType, depth: usize) -> Result { if let Some(delegate) = ch_type.physical_delegate_ref() { - return Self::new(delegate.as_ref()); + return Self::new( + delegate.as_ref(), + depth + usize::from(matches!(ch_type, ChType::SimpleAggregateFunction { .. })), + ); } Ok(match ch_type { ChType::Nothing => Self::Nothing { rows: 0 }, @@ -336,7 +359,7 @@ impl ValueBuilder { element_type, dimension, } => Self::QBit { - values: Box::new(Self::new(&element_type.ch_type())?), + values: Box::new(Self::new(&element_type.ch_type(), depth + 1)?), dimension: *dimension, }, ChType::Date => Self::Date(Vec::new()), @@ -379,13 +402,13 @@ impl ValueBuilder { } ChType::Nullable(inner) => Self::Nullable { flags: Vec::new(), - inner: Box::new(Self::new(inner)?), + inner: Box::new(Self::new(inner, depth + 1)?), }, // A LowCardinality single value carries no dictionary framing; the // decoded column is the plain inner column. - ChType::LowCardinality(inner) => Self::new(inner)?, + ChType::LowCardinality(inner) => Self::new(inner, depth + 1)?, ChType::Array(inner) => { - let elements = Self::new(inner)?; + let elements = Self::new(inner, depth + 1)?; Self::Array { offsets: vec![0], fill: elements.is_zero_width(), @@ -394,7 +417,7 @@ impl ValueBuilder { } } ChType::Map(key, value) => { - let (keys, values) = (Self::new(key)?, Self::new(value)?); + let (keys, values) = (Self::new(key, depth + 1)?, Self::new(value, depth + 1)?); Self::Map { offsets: vec![0], fill: keys.is_zero_width() && values.is_zero_width(), @@ -406,17 +429,30 @@ impl ValueBuilder { ChType::Tuple(elements) => Self::Tuple { fields: elements .iter() - .map(|(_, t)| Self::new(t)) + .map(|(_, t)| Self::new(t, depth + 1)) .collect::>()?, rows: 0, }, - // JSON values are not materialized here: shared-data and - // SharedVariant cells stay opaque, so a JSON-typed binary value is - // reported unsupported rather than partially decoded. - ChType::Variant(_) - | ChType::Dynamic { .. } - | ChType::Json { .. } - | ChType::AggregateFunction { .. } => { + ChType::Json { typed_paths, .. } => { + if typed_paths.windows(2).any(|pair| pair[0].0 >= pair[1].0) { + return Err(BinaryValueError::Invalid( + "JSON typed paths are not strictly sorted".into(), + )); + } + Self::Json(Box::new(JsonValueBuilder { + typed: typed_paths + .iter() + .map(|(path, ty)| Ok((path.clone(), Self::new(ty, depth + 1)?))) + .collect::>()?, + offsets: vec![0], + paths: Utf8Column::new(vec![0], Vec::new()), + values: Utf8Column::new(vec![0], Vec::new()), + depth, + seen: vec![false; typed_paths.len()], + path_order: Vec::new(), + })) + } + ChType::Variant(_) | ChType::Dynamic { .. } | ChType::AggregateFunction { .. } => { return Err(BinaryValueError::Unsupported(ch_type.to_string())) } // physical_delegate expanded these above. @@ -474,7 +510,7 @@ impl ValueBuilder { // The server cannot serialize a Nothing value; only null rows // (which take the default path) are representable. ValueBuilder::Nothing { .. } => { - return Err(BinaryValueError::Unsupported("Nothing".into())) + return Err(BinaryValueError::Invalid("non-null Nothing value".into())) } ValueBuilder::Bool(values) => values.push(reader.read_u8()?), ValueBuilder::Int8(values) => values.push(reader.read_u8()? as i8), @@ -582,6 +618,65 @@ impl ValueBuilder { } *rows += 1; } + ValueBuilder::Json(json) => { + let JsonValueBuilder { + typed, + offsets, + paths, + values, + depth, + seen, + path_order, + } = json.as_mut(); + let row_start = reader.position(); + let row_bytes = reader.remaining_slice(); + let count = read_element_count(reader, false, "JSON path")?; + seen.fill(false); + let mut missing_typed = typed.len(); + path_order.clear(); + for _ in 0..count { + let path = read_json_path(reader)?; + let path_end = reader.position() - row_start; + if let Ok(index) = typed.binary_search_by(|(name, _)| name.as_bytes().cmp(path)) + { + if seen[index] { + return Err(BinaryValueError::Invalid( + "duplicate JSON typed path".into(), + )); + } + seen[index] = true; + missing_typed -= 1; + typed[index].1.append_value(reader, null_budget)?; + } else { + let start = reader.position(); + let ty = read_value_type(reader, *depth + 1)?; + ty.skip(reader)?; + path_order.push(( + path_end - path.len()..path_end, + start - row_start..reader.position() - row_start, + )); + } + } + path_order + .sort_unstable_by(|a, b| row_bytes[a.0.clone()].cmp(&row_bytes[b.0.clone()])); + if path_order + .windows(2) + .any(|pair| row_bytes[pair[0].0.clone()] == row_bytes[pair[1].0.clone()]) + { + return Err(BinaryValueError::Invalid("duplicate JSON path".into())); + } + for (path, cell) in path_order.iter() { + append_utf8(paths, &row_bytes[path.clone()])?; + append_utf8(values, &row_bytes[cell.clone()])?; + } + null_budget.charge(missing_typed)?; + for ((_, builder), seen) in typed.iter_mut().zip(seen.iter()) { + if !seen { + builder.append_default(null_budget)?; + } + } + push_end_offset(offsets, path_order.len())?; + } } Ok(()) } @@ -594,7 +689,10 @@ impl ValueBuilder { ) -> Result<(), BinaryValueError> { count_visit(); match self { - ValueBuilder::Nothing { rows } => *rows += 1, + ValueBuilder::Nothing { rows } => { + null_budget.charge(1)?; + *rows += 1; + } ValueBuilder::Bool(values) => { null_budget.charge(1)?; values.push(0); @@ -727,11 +825,21 @@ impl ValueBuilder { offsets.push(last); } ValueBuilder::Tuple { fields, rows } => { + null_budget.charge(1)?; for field in fields.iter_mut() { field.append_default(null_budget)?; } *rows += 1; } + ValueBuilder::Json(json) => { + let JsonValueBuilder { typed, offsets, .. } = json.as_mut(); + null_budget.charge(typed.len())?; + null_budget.charge(8)?; + for (_, field) in typed { + field.append_default(null_budget)?; + } + push_end_offset(offsets, 0)?; + } } Ok(()) } @@ -846,6 +954,31 @@ impl ValueBuilder { )); Column::Map(MapColumn::new(offsets, entries)) } + ValueBuilder::Json(json) => { + let JsonValueBuilder { + typed, + offsets, + paths, + values, + .. + } = *json; + let len = offsets.len() - 1; + let typed = typed + .into_iter() + .map(|(path, builder)| Ok((path, builder.finish(None)?))) + .collect::>()?; + Column::Json( + JsonColumn::structured(StructuredJson::from_parts( + typed, + Vec::new(), + offsets, + paths, + values, + len, + )) + .with_validity(validity), + ) + } ValueBuilder::Tuple { fields, rows } => { let columns = fields .into_iter() @@ -871,6 +1004,275 @@ fn push_end_offset(offsets: &mut Vec, count: usize) -> Result<(), BinaryVal Ok(()) } +fn append_utf8(column: &mut Utf8Column, bytes: &[u8]) -> Result<(), BinaryValueError> { + let end = column + .data + .len() + .checked_add(bytes.len()) + .and_then(|end| i32::try_from(end).ok()) + .ok_or_else(|| BinaryValueError::Invalid("JSON data exceeds i32 offsets".into()))?; + column.data.extend_from_slice(bytes); + column.offsets.push(end); + Ok(()) +} + +fn read_json_path<'a>(reader: &mut ByteReader<'a>) -> Result<&'a [u8], BinaryValueError> { + let len = usize::try_from(reader.read_varint()?) + .map_err(|_| BinaryValueError::Invalid("JSON path length overflows usize".into()))?; + Ok(reader.read_slice(len)?) +} + +fn read_value_type(reader: &mut ByteReader, depth: usize) -> Result { + let ty = read_binary_type(reader).map_err(|error| match error { + BinaryTypeError::Unsupported(reason) => BinaryValueError::Unsupported(reason), + BinaryTypeError::Invalid(reason) => BinaryValueError::Invalid(reason), + BinaryTypeError::Io(error) => error.into(), + })?; + check_type(&ty, None, depth).ok_or_else(too_deep)?; + if ty == ChType::Nothing { + // Dynamic NULL has only its Nothing descriptor. + Ok(SkipPlan::Fixed(0)) + } else { + SkipPlan::new(&ty, depth) + } +} + +#[cfg(test)] +thread_local! { + static SKIP_STEPS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Compiled row framing for one binary descriptor. +/// +/// At v26.8.1.2041-lts, SerializationObject::serializeBinary writes a path +/// count, then path/value pairs. Declared typed paths use their serializer; +/// every other value starts with a DataTypesBinaryEncoding descriptor. +enum SkipPlan { + Nothing, + Fixed(usize), + String, + Nullable(Box), + Array(Box), + Map(Box), + Tuple(Vec), + QBit { + dimension: usize, + bytes: usize, + }, + Json { + typed: Vec<(String, SkipPlan)>, + depth: usize, + }, + Dynamic { + depth: usize, + }, + Variant(Vec), + Unsupported(String), +} + +impl SkipPlan { + fn new(ch_type: &ChType, depth: usize) -> Result { + if depth > MAX_TYPE_DEPTH { + return Err(too_deep()); + } + if let Some(delegate) = ch_type.physical_delegate_ref() { + return Self::new( + delegate.as_ref(), + depth + usize::from(matches!(ch_type, ChType::SimpleAggregateFunction { .. })), + ); + } + let width = match ch_type { + ChType::Nothing => return Ok(Self::Nothing), + ChType::Bool | ChType::Int8 | ChType::UInt8 | ChType::Enum8 { .. } => 1, + ChType::Int16 + | ChType::UInt16 + | ChType::Date + | ChType::Enum16 { .. } + | ChType::BFloat16 => 2, + ChType::Int32 + | ChType::UInt32 + | ChType::Float32 + | ChType::Date32 + | ChType::DateTime { .. } + | ChType::Time + | ChType::Ipv4 => 4, + ChType::Int64 + | ChType::UInt64 + | ChType::Float64 + | ChType::DateTime64 { .. } + | ChType::Time64 { .. } + | ChType::Interval(_) => 8, + ChType::Int128 | ChType::UInt128 | ChType::Uuid | ChType::Ipv6 => 16, + ChType::Int256 | ChType::UInt256 => 32, + ChType::Decimal { bits, .. } if matches!(bits, 32 | 64 | 128 | 256) => { + usize::from(*bits / 8) + } + ChType::FixedString(width) if *width != 0 => *width, + ChType::String => return Ok(Self::String), + ChType::Nullable(inner) => { + return Ok(Self::Nullable(Box::new(Self::new(inner, depth + 1)?))) + } + ChType::LowCardinality(inner) => return Self::new(inner, depth + 1), + ChType::Array(inner) => return Ok(Self::Array(Box::new(Self::new(inner, depth + 1)?))), + ChType::Map(key, value) => { + return Ok(Self::Map(Box::new(Self::sequence([ + Self::new(key, depth + 1)?, + Self::new(value, depth + 1)?, + ])?))) + } + ChType::Tuple(fields) => { + return Self::sequence( + fields + .iter() + .map(|(_, ty)| Self::new(ty, depth + 1)) + .collect::, _>>()?, + ) + } + ChType::QBit { + element_type, + dimension, + } => { + let bytes = dimension + .checked_mul(element_type.byte_width()) + .ok_or_else(|| { + BinaryValueError::Invalid("QBit payload size overflows usize".into()) + })?; + return Ok(Self::QBit { + dimension: *dimension, + bytes, + }); + } + ChType::Json { typed_paths, .. } => { + return Ok(Self::Json { + typed: typed_paths + .iter() + .map(|(path, ty)| Ok((path.clone(), Self::new(ty, depth + 1)?))) + .collect::>()?, + depth, + }) + } + ChType::Dynamic { .. } => return Ok(Self::Dynamic { depth }), + ChType::Variant(alternatives) => { + return Ok(Self::Variant( + alternatives + .iter() + .map(|ty| Self::new(ty, depth + 1)) + .collect::>()?, + )) + } + ChType::AggregateFunction { .. } => return Ok(Self::Unsupported(ch_type.to_string())), + _ => { + return Err(BinaryValueError::Invalid( + "invalid binary value type".into(), + )) + } + }; + Ok(Self::Fixed(width)) + } + + fn sequence(parts: impl IntoIterator) -> Result { + let mut sequence = Vec::new(); + let mut width = 0usize; + for part in parts { + if let Self::Fixed(bytes) = part { + width = width.checked_add(bytes).ok_or_else(|| { + BinaryValueError::Invalid("tuple payload size overflows usize".into()) + })?; + } else { + if width != 0 { + sequence.push(Self::Fixed(width)); + width = 0; + } + sequence.push(part); + } + } + if width != 0 { + sequence.push(Self::Fixed(width)); + } + Ok(match sequence.len() { + 0 => Self::Fixed(0), + 1 => sequence.remove(0), + _ => Self::Tuple(sequence), + }) + } + + fn skip(&self, reader: &mut ByteReader) -> Result<(), BinaryValueError> { + #[cfg(test)] + SKIP_STEPS.with(|steps| steps.set(steps.get() + 1)); + match self { + Self::Nothing => { + return Err(BinaryValueError::Invalid("non-null Nothing value".into())) + } + Self::Fixed(bytes) => reader.skip(*bytes)?, + Self::String => { + let len = usize::try_from(reader.read_varint()?).map_err(|_| { + BinaryValueError::Invalid("String length overflows usize".into()) + })?; + reader.skip(len)?; + } + Self::Nullable(inner) => { + if reader.read_u8()? == 0 { + inner.skip(reader)?; + } + } + Self::Array(inner) | Self::Map(inner) => { + let what = if matches!(self, Self::Array(_)) { + "Array element" + } else { + "Map entry" + }; + let count = + read_element_count(reader, matches!(inner.as_ref(), Self::Fixed(0)), what)?; + if let Self::Fixed(width) = inner.as_ref() { + let bytes = count.checked_mul(*width).ok_or_else(|| { + BinaryValueError::Invalid("container payload size overflows usize".into()) + })?; + reader.skip(bytes)?; + } else { + for _ in 0..count { + inner.skip(reader)?; + } + } + } + Self::Tuple(fields) => { + for field in fields { + field.skip(reader)?; + } + } + Self::QBit { dimension, bytes } => { + if reader.read_varint()? != *dimension as u64 { + return Err(BinaryValueError::Invalid("QBit dimension mismatch".into())); + } + reader.skip(*bytes)?; + } + Self::Json { typed, depth } => { + let count = read_element_count(reader, false, "JSON path")?; + for _ in 0..count { + let path = read_json_path(reader)?; + match typed.binary_search_by(|(name, _)| name.as_bytes().cmp(path)) { + Ok(index) => typed[index].1.skip(reader)?, + Err(_) => read_value_type(reader, depth + 1)?.skip(reader)?, + } + } + } + Self::Dynamic { depth } => read_value_type(reader, depth + 1)?.skip(reader)?, + Self::Variant(alternatives) => { + let discriminator = reader.read_u8()?; + if discriminator != u8::MAX { + let plan = alternatives + .get(usize::from(discriminator)) + .ok_or_else(|| { + BinaryValueError::Invalid("invalid Variant discriminator".into()) + })?; + plan.skip(reader)?; + } + } + Self::Unsupported(name) => return Err(BinaryValueError::Unsupported(name.clone())), + } + Ok(()) + } +} + // --------------------------------------------------------------------------- // Encoding // --------------------------------------------------------------------------- @@ -935,9 +1337,9 @@ pub fn write_binary_type_prefix( /// shared data. Nullable flags are for nested values, such as elements of /// `Array(Nullable(T))`. /// -/// `Variant`, `Dynamic`, `JSON`, `AggregateFunction`, and non-null `Nothing` -/// values are `Unsupported`, matching the decoder. A row out of range or a -/// column whose shape does not match `ch_type`, or a null row under a +/// `Variant`, `Dynamic`, and `AggregateFunction` values are `Unsupported`, +/// matching the decoder. Non-null `Nothing` values are `Invalid`. A row out of +/// range or a column whose shape does not match `ch_type`, or a null row under a /// non-nullable type, is `Invalid`. The column is caller memory rather than /// wire input, but malformed buffers still return an error instead of panicking. /// On error `buf` is left as it was. @@ -955,7 +1357,7 @@ pub fn encode_binary_value( return Err(mismatch(ch_type)); } let start = buf.len(); - let result = encode_value(buf, ch_type, column, row); + let result = encode_value(buf, ch_type, column, row, 0); if result.is_err() { buf.truncate(start); } @@ -968,6 +1370,7 @@ pub fn encode_binary_value( struct TypeCheck { supported: bool, matches: bool, + descriptor_nodes: usize, } impl TypeCheck { @@ -975,20 +1378,29 @@ impl TypeCheck { fn combine(&mut self, other: Self) { self.supported &= other.supported; self.matches &= other.matches; + self.descriptor_nodes = self.descriptor_nodes.saturating_add(other.descriptor_nodes); + } + + fn check_descriptor_complexity(self) -> Result<(), BinaryValueError> { + if self.descriptor_nodes > MAX_BINARY_TYPE_COMPLEXITY { + return Err(BinaryValueError::Invalid(format!( + "complexity exceeds {MAX_BINARY_TYPE_COMPLEXITY}" + ))); + } + Ok(()) } } -/// Check depth, coverage, and optional column shape in one allocation-free -/// traversal. None means too deep. Every type child is visited even if its +/// Check depth, coverage, descriptor nodes, and optional column shape without +/// allocations. None means too deep. Every type child is visited even if its /// column shape is already wrong, before any error can render the type. /// Depth charges match `encode::validate::type_depth`, including empty aliases. /// -/// Single-child wrappers (`Nullable`, `LowCardinality`, `Array`, and the -/// `SimpleAggregateFunction` alias) are walked in a loop, so a chain like +/// Single-child wrappers (`Nullable`, `LowCardinality`, and `Array`) are walked +/// in a loop, so a chain like /// `Array(LowCardinality(Nullable(String)))` costs one frame and a hostile -/// million-deep chain is rejected without recursion. Only branching nodes -/// (`Tuple`, `Map`, `Nested`, and the unsupported families) recurse, and the -/// depth gate bounds that recursion. +/// million-deep chain is rejected without recursion. Branching nodes and +/// `SimpleAggregateFunction` recurse, bounded by the depth gate. #[inline] fn check_type( mut ch_type: &ChType, @@ -998,15 +1410,24 @@ fn check_type( let mut check = TypeCheck { supported: true, matches: true, + descriptor_nodes: 0, }; loop { if depth > MAX_TYPE_DEPTH { return None; } + check.descriptor_nodes = check.descriptor_nodes.saturating_add(1); match ch_type { - ChType::Nullable(inner) | ChType::SimpleAggregateFunction { inner, .. } => { + ChType::Nullable(inner) => { ch_type = inner; } + ChType::SimpleAggregateFunction { inner, .. } => { + let inner = check_type(inner, column, depth + 1)?; + check.supported &= inner.supported; + check.matches &= inner.matches; + // The descriptor writer uses one Custom tag for this alias. + return Some(check); + } ChType::LowCardinality(inner) => { if let Some(Column::Dictionary(dict)) = column { column = Some(dict.values.as_ref()); @@ -1113,7 +1534,7 @@ fn check_type( } ChType::Variant(alternatives) => { for ty in alternatives { - check_type(ty, None, depth + 1)?; + check.combine(check_type(ty, None, depth + 1)?); } check.supported = false; return Some(check); @@ -1126,10 +1547,38 @@ fn check_type( return Some(check); } ChType::Json { typed_paths, .. } => { - for (_, ty) in typed_paths { - check_type(ty, None, depth + 1)?; + let body = match column { + Some(Column::Json(JsonColumn { + body: JsonBody::Structured(body), + .. + })) => Some(body.as_ref()), + Some(Column::Json(_)) => { + check.supported = false; + None + } + Some(_) => { + check.matches = false; + None + } + None => None, + }; + check.matches &= !typed_paths.windows(2).any(|pair| pair[0].0 >= pair[1].0); + if let Some(body) = body { + check.matches &= body.typed.len() == typed_paths.len() + && !body.typed.windows(2).any(|pair| pair[0].0 >= pair[1].0); + } + for (path, ty) in typed_paths { + let child = body.and_then(|body| { + body.typed + .binary_search_by(|(name, _)| name.as_bytes().cmp(path.as_bytes())) + .ok() + .map(|index| &body.typed[index].1) + }); + if body.is_some() && child.is_none() { + check.matches = false; + } + check.combine(check_type(ty, child, depth + 1)?); } - check.supported = false; return Some(check); } ChType::Dynamic { .. } => { @@ -1137,6 +1586,9 @@ fn check_type( return Some(check); } _ => { + if matches!(ch_type, ChType::QBit { .. }) { + check.descriptor_nodes = check.descriptor_nodes.saturating_add(1); + } if let Some(column) = column { check.matches &= scalar_variant_matches(ch_type, column); } @@ -1222,7 +1674,7 @@ fn scalar_variant_matches(ch_type: &ChType, column: &Column) -> bool { #[cold] #[inline(never)] fn too_deep() -> BinaryValueError { - BinaryValueError::Invalid("type nesting too deep".into()) + BinaryValueError::Invalid(format!("nesting exceeds {MAX_TYPE_DEPTH}")) } #[cold] @@ -1367,6 +1819,137 @@ fn fixed_slice(data: &[u8], width: usize, row: usize) -> Result<&[u8], BinaryVal .ok_or_else(|| out_of_range(row, data.len().checked_div(width).unwrap_or(0))) } +fn utf8_slice(column: &Utf8Column, row: usize) -> Result<&[u8], BinaryValueError> { + reject_null(column.validity.as_ref(), row)?; + let (start, end) = offset_range(&column.offsets, row)?; + column + .data + .get(start..end) + .ok_or_else(|| BinaryValueError::Invalid("invalid JSON string offsets".into())) +} + +fn write_string(buf: &mut Vec, bytes: &[u8]) { + write_varint(buf, bytes.len() as u64); + buf.extend_from_slice(bytes); +} + +fn write_shared_value( + buf: &mut Vec, + cell: &[u8], + depth: usize, +) -> Result<(), BinaryValueError> { + let mut reader = ByteReader::new(cell); + let ty = read_value_type(&mut reader, depth)?; + ty.skip(&mut reader)?; + if reader.remaining() != 0 { + return Err(BinaryValueError::Invalid( + "trailing JSON shared value bytes".into(), + )); + } + buf.extend_from_slice(cell); + Ok(()) +} + +fn encode_json_value( + buf: &mut Vec, + typed_paths: &[(String, ChType)], + column: &JsonColumn, + row: usize, + depth: usize, +) -> Result<(), BinaryValueError> { + reject_null(column.validity.as_ref(), row)?; + let JsonBody::Structured(body) = &column.body else { + return Err(BinaryValueError::Unsupported("JSON text body".into())); + }; + if row >= body.len { + return Err(out_of_range(row, body.len)); + } + let (start, end) = offset_range(&body.shared_offsets, row)?; + if end > body.shared_paths.len() || end > body.shared_values.len() { + return Err(BinaryValueError::Invalid( + "invalid JSON shared offsets".into(), + )); + } + let mut count = typed_paths + .len() + .checked_add(end - start) + .ok_or_else(|| BinaryValueError::Invalid("JSON path count overflows usize".into()))?; + for (_, dynamic) in &body.dynamic { + let id = dynamic + .type_ids + .get(row) + .ok_or_else(|| out_of_range(row, dynamic.len()))?; + if *id != u32::MAX { + count = count.checked_add(1).ok_or_else(|| { + BinaryValueError::Invalid("JSON path count overflows usize".into()) + })?; + } + } + let mut paths = Vec::with_capacity(count); + paths.extend(body.typed.iter().map(|(path, _)| path.as_bytes())); + for (path, dynamic) in &body.dynamic { + if dynamic.type_ids[row] != u32::MAX { + paths.push(path.as_bytes()); + } + } + for index in start..end { + let path = utf8_slice(&body.shared_paths, index)?; + paths.push(path); + } + paths.sort_unstable(); + if paths.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(BinaryValueError::Invalid("duplicate JSON path".into())); + } + write_varint(buf, count as u64); + for (path, values) in &body.typed { + let index = typed_paths + .binary_search_by(|(name, _)| name.as_bytes().cmp(path.as_bytes())) + .map_err(|_| BinaryValueError::Invalid("unexpected JSON typed path".into()))?; + let ty = &typed_paths[index].1; + write_string(buf, path.as_bytes()); + encode_value(buf, ty, values, row, depth + 1)?; + } + for (path, dynamic) in &body.dynamic { + let id = *dynamic + .type_ids + .get(row) + .ok_or_else(|| out_of_range(row, dynamic.len()))?; + if id == u32::MAX { + continue; + } + let offset = dynamic + .offsets + .get(row) + .and_then(|offset| usize::try_from(*offset).ok()) + .ok_or_else(|| BinaryValueError::Invalid("invalid JSON dynamic offset".into()))?; + let child = dynamic + .children + .get(id as usize) + .ok_or_else(|| BinaryValueError::Invalid("invalid JSON dynamic type id".into()))?; + write_string(buf, path.as_bytes()); + match child { + DynamicChild::Typed { ch_type, values } => { + let check = check_type(ch_type, Some(values), depth + 1).ok_or_else(too_deep)?; + check.check_descriptor_complexity()?; + if !check.matches { + return Err(mismatch(ch_type)); + } + write_binary_type(buf, ch_type); + encode_value(buf, ch_type, values, offset, depth + 1)?; + } + DynamicChild::Shared(values) => { + write_shared_value(buf, utf8_slice(values, offset)?, depth + 1)?; + } + } + } + for index in start..end { + let path = utf8_slice(&body.shared_paths, index)?; + write_string(buf, path); + write_shared_value(buf, utf8_slice(&body.shared_values, index)?, depth + 1)?; + } + Ok(()) +} + /// One row of a `Tuple` column under `elements`: shared by `Tuple` and the /// `Nested` alias, whose elements are an `Array` of this shape. #[inline] @@ -1376,6 +1959,7 @@ fn encode_tuple<'a>( elements: impl ExactSizeIterator, c: &TupleColumn, row: usize, + depth: usize, ) -> Result<(), BinaryValueError> { if elements.len() != c.fields.len() { return Err(mismatch(ch_type)); @@ -1385,7 +1969,7 @@ fn encode_tuple<'a>( return Err(out_of_range(row, c.len)); } for (element, field) in elements.zip(&c.fields) { - encode_value(buf, element, field, row)?; + encode_value(buf, element, field, row, depth + 1)?; } Ok(()) } @@ -1398,8 +1982,12 @@ fn encode_value( ch_type: &ChType, column: &Column, row: usize, + depth: usize, ) -> Result<(), BinaryValueError> { count_visit(); + if depth > MAX_TYPE_DEPTH { + return Err(too_deep()); + } macro_rules! le { ($c:expr) => {{ reject_null($c.validity.as_ref(), row)?; @@ -1415,9 +2003,11 @@ fn encode_value( // Name-decoration aliases encode as their physical type. `Nested` has // its own arm below so its `Array(Tuple(..))` form is never built. (ChType::SimpleAggregateFunction { inner, .. }, _) => { - encode_value(buf, inner, column, row)? + encode_value(buf, inner, column, row, depth + 1)? + } + (ChType::Geo(kind), _) => { + encode_value(buf, kind.underlying_type_ref(), column, row, depth)? } - (ChType::Geo(kind), _) => encode_value(buf, kind.underlying_type_ref(), column, row)?, // Mirror the bulk null map: 0x00 then the value, or 0x01 and nothing. (ChType::Nullable(inner), _) => { @@ -1428,7 +2018,7 @@ fn encode_value( buf.push(1); } else { buf.push(0); - encode_value(buf, inner, column, row)?; + encode_value(buf, inner, column, row, depth + 1)?; } } (ChType::LowCardinality(inner), Column::Dictionary(dict)) => { @@ -1452,14 +2042,14 @@ fn encode_value( let index = usize::try_from(index).map_err(|_| { BinaryValueError::Invalid(format!("row {row} has negative dictionary index")) })?; - encode_value(buf, value_type, &dict.values, index)?; + encode_value(buf, value_type, &dict.values, index, depth + 1)?; } // The plain inner column, as decode_binary_value returns it. - (ChType::LowCardinality(inner), _) => encode_value(buf, inner, column, row)?, + (ChType::LowCardinality(inner), _) => encode_value(buf, inner, column, row, depth + 1)?, (ChType::Nothing, Column::Nothing(c)) => { reject_null(c.validity.as_ref(), row)?; - return Err(BinaryValueError::Unsupported("Nothing".into())); + return Err(BinaryValueError::Invalid("non-null Nothing value".into())); } (ChType::Bool, Column::Bool(c)) => { reject_null(c.validity.as_ref(), row)?; @@ -1595,7 +2185,7 @@ fn encode_value( write_varint(buf, (end - start) as u64); let from = first_checked_row(start, end, || zero_width_bound(inner, &c.values, end)); for i in from..end { - encode_value(buf, inner, &c.values, i)?; + encode_value(buf, inner, &c.values, i, depth + 1)?; } } (ChType::Nested(fields), Column::Array(c)) => { @@ -1614,7 +2204,7 @@ fn encode_value( zero_width_tuple_bound(elements(), tuple, end) }); for i in from..end { - encode_tuple(buf, ch_type, elements(), tuple, i)?; + encode_tuple(buf, ch_type, elements(), tuple, i, depth + 1)?; } } (ChType::Map(key, value), Column::Map(c)) => { @@ -1634,17 +2224,24 @@ fn encode_value( Some(key.min(zero_width_bound(value, values, end)?)) }); for i in from..end { - encode_value(buf, key, keys, i)?; - encode_value(buf, value, values, i)?; + encode_value(buf, key, keys, i, depth + 1)?; + encode_value(buf, value, values, i, depth + 1)?; } } - (ChType::Tuple(elements), Column::Tuple(c)) => { - encode_tuple(buf, ch_type, elements.iter().map(|(_, ty)| ty), c, row)? + (ChType::Tuple(elements), Column::Tuple(c)) => encode_tuple( + buf, + ch_type, + elements.iter().map(|(_, ty)| ty), + c, + row, + depth, + )?, + (ChType::Json { typed_paths, .. }, Column::Json(c)) => { + encode_json_value(buf, typed_paths, c, row, depth)?; } ( ChType::Variant(_) | ChType::Dynamic { .. } - | ChType::Json { .. } | ChType::AggregateFunction { .. } | ChType::Geometry, _, @@ -2037,6 +2634,596 @@ mod tests { assert_eq!(c.values, vec![79]); } + #[test] + fn json_typed_paths_use_names_and_default_missing_values() { + let ty = parse("Array(JSON(a Int64, b Nullable(String)))"); + let mut bytes = vec![2, 2]; + write_string(&mut bytes, b"b"); + bytes.push(0); + write_string(&mut bytes, b"user_1"); + write_string(&mut bytes, b"a"); + bytes.extend_from_slice(&13i64.to_le_bytes()); + bytes.push(0); + let column = decode_binary_value(&ty, &bytes).unwrap(); + let Column::Array(array) = &column else { + panic!("expected Array") + }; + let Column::Json(json) = array.values.as_ref() else { + panic!("expected JSON") + }; + let JsonBody::Structured(body) = &json.body else { + panic!("expected structured") + }; + let Column::Int64(a) = &body.typed[0].1 else { + panic!("expected Int64") + }; + assert_eq!(a.values, [13, 0]); + let Column::Utf8(b) = &body.typed[1].1 else { + panic!("expected String") + }; + assert_eq!(b.value(0), b"user_1"); + assert_eq!(b.value(1), b""); + assert_eq!(b.validity.as_ref().unwrap().null_count(), 1); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(decode_binary_value(&ty, &encoded).unwrap(), column); + } + + #[test] + fn json_nullable_defaults_share_the_materialization_budget() { + let ty = parse("Nullable(JSON(a FixedString(16777215), b FixedString(16777215)))"); + assert!(matches!( + decode_binary_value(&ty, &[1]), + Err(BinaryValueError::Invalid(reason)) if reason.contains("placeholder expansion") + )); + let ty = parse("Nullable(JSON(a Nullable(Int64)))"); + let column = decode_binary_value(&ty, &[1]).unwrap(); + assert_eq!(column.null_count(), 1); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, [1]); + assert!(matches!( + decode_binary_value(&parse("JSON(v Variant(Int64, String))"), &[0]), + Err(BinaryValueError::Unsupported(_)) + )); + } + + #[test] + fn json_missing_zero_width_defaults_charge_the_budget() { + for name in [ + "JSON(a Tuple(), b Tuple())", + "JSON(a Tuple(Tuple(), Tuple()))", + "JSON(a Tuple(Nothing, Nothing))", + ] { + let mut builder = ValueBuilder::new(&parse(name), 0).unwrap(); + let mut reader = ByteReader::new(&[0]); + let mut budget = NullDefaultBudget { remaining: 1 }; + assert!( + matches!( + builder.append_value(&mut reader, &mut budget), + Err(BinaryValueError::Invalid(reason)) if reason.contains("placeholder expansion") + ), + "{name}" + ); + } + } + + #[test] + fn json_wide_missing_defaults_stop_before_exhausting_the_input() { + for (count, value_type) in [ + (999, ChType::Tuple(Vec::new())), + (1000, ChType::Tuple(vec![(None, ChType::Tuple(Vec::new()))])), + ] { + let json = ChType::Json { + max_dynamic_paths: 0, + max_dynamic_types: 0, + typed_paths: (0..count) + .map(|index| (format!("p{index:04}"), value_type.clone())) + .collect(), + skip_paths: Vec::new(), + skip_regexps: Vec::new(), + }; + for (inner, row) in [(json.clone(), 0), (ChType::Nullable(Box::new(json)), 1)] { + let ty = ChType::Array(Box::new(inner)); + let mut bytes = varint(1_000_000); + bytes.resize(bytes.len() + 1_000_000, row); + assert!(matches!( + decode_binary_value(&ty, &bytes), + Err(BinaryValueError::Invalid(reason)) if reason.contains("placeholder expansion") + )); + } + } + } + + #[test] + fn json_shared_cells_reject_non_null_nothing_values() { + for (name, payload) in [ + ("Array(Nothing)", vec![1]), + ("Array(Nothing)", varint(1_000_000)), + ("Array(Nothing)", varint(u64::MAX)), + ("Nullable(Nothing)", vec![0]), + ("Tuple(Nothing)", vec![]), + ("Map(Nothing, UInt8)", vec![1, 79]), + ("Map(String, Nothing)", vec![1, 1, b'k']), + ] { + let mut cell = Vec::new(); + write_binary_type(&mut cell, &parse(name)); + cell.extend(payload); + let mut bytes = vec![1]; + write_string(&mut bytes, b"x"); + bytes.extend_from_slice(&cell); + assert!( + matches!( + decode_binary_value(&parse("JSON"), &bytes), + Err(BinaryValueError::Invalid(_)) + ), + "{name}" + ); + let column = Column::Json(JsonColumn::structured(StructuredJson::from_parts( + Vec::new(), + Vec::new(), + vec![0, 1], + Utf8Column::new(vec![0, 1], vec![b'x']), + Utf8Column::new(vec![0, cell.len() as i32], cell), + 1, + ))); + let mut encoded = vec![0xab]; + assert!( + matches!( + encode_binary_value(&mut encoded, &parse("JSON"), &column, 0), + Err(BinaryValueError::Invalid(_)) + ), + "{name}" + ); + assert_eq!(encoded, [0xab]); + } + } + + #[test] + fn json_shared_cells_allow_nothing_only_when_null_or_empty() { + for (name, payload) in [ + ("Nothing", vec![]), + ("Array(Nothing)", vec![0]), + ("Map(Nothing, UInt8)", vec![0]), + ("Map(String, Nothing)", vec![0]), + ("Nullable(Nothing)", vec![1]), + ("Array(Nullable(Nothing))", vec![3, 1, 1, 1]), + ("Array(Array(Nothing))", vec![2, 0, 0]), + ("Array(Dynamic)", vec![3, 0, 0, 0]), + ("Variant(String, UInt8)", vec![255]), + ] { + let mut bytes = vec![1]; + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &parse(name)); + bytes.extend(payload); + let column = decode_binary_value(&parse("JSON"), &bytes) + .unwrap_or_else(|error| panic!("{name}: {error}")); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &parse("JSON"), &column, 0).unwrap(); + assert_eq!(encoded, bytes, "{name}"); + } + } + + #[test] + fn json_rejects_truncation_trailing_bytes_and_duplicate_typed_paths() { + let ty = parse("JSON(a Int64)"); + let mut bytes = vec![2]; + write_string(&mut bytes, b"a"); + bytes.extend_from_slice(&13i64.to_le_bytes()); + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &parse("Array(Nullable(String))")); + bytes.extend_from_slice(&[2, 1, 0]); + write_string(&mut bytes, b"user_1"); + assert!(decode_binary_value(&ty, &bytes).is_ok()); + for end in 0..bytes.len() { + assert!( + matches!( + decode_binary_value(&ty, &bytes[..end]), + Err(BinaryValueError::Invalid(_)) + ), + "{end}" + ); + } + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + decode_binary_value(&ty, &trailing), + Err(BinaryValueError::Invalid(_)) + )); + let mut duplicate = vec![2]; + for _ in 0..2 { + write_string(&mut duplicate, b"a"); + duplicate.extend_from_slice(&13i64.to_le_bytes()); + } + assert!(matches!( + decode_binary_value(&ty, &duplicate), + Err(BinaryValueError::Invalid(_)) + )); + assert!(matches!( + decode_binary_value(&parse("JSON"), &[1, 1, 255]), + Err(BinaryValueError::Invalid(_)) + )); + } + + #[test] + fn json_embedded_descriptors_share_the_value_depth_limit() { + let ty = parse("JSON"); + let descriptor = { + let mut bytes = Vec::new(); + write_binary_type(&mut bytes, &ty); + bytes + }; + for (depth, valid) in [(5, true), (MAX_TYPE_DEPTH + 1, false)] { + let mut bytes = Vec::new(); + for _ in 0..depth { + bytes.extend_from_slice(&[1, 1, b'x']); + bytes.extend_from_slice(&descriptor); + } + bytes.push(0); + let decoded = decode_binary_value(&ty, &bytes); + assert_eq!(decoded.is_ok(), valid); + if let Ok(column) = decoded { + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes); + } + } + } + + #[test] + fn json_rejects_colliding_writer_paths_and_repeated_shared_reader_paths() { + let ty = parse("JSON(a Int64)"); + for (dynamic_names, shared_names) in [ + (&["a"][..], &[][..]), + (&[][..], &["a"][..]), + (&["x"][..], &["x"][..]), + (&["x", "x"][..], &[][..]), + (&[][..], &["x", "x"][..]), + ] { + let mut paths = Utf8Column::new(vec![0], Vec::new()); + let mut values = Utf8Column::new(vec![0], Vec::new()); + let mut cell = Vec::new(); + write_binary_type(&mut cell, &ChType::UInt8); + cell.push(79); + for path in shared_names { + append_utf8(&mut paths, path.as_bytes()).unwrap(); + append_utf8(&mut values, &cell).unwrap(); + } + let dynamic = dynamic_names + .iter() + .map(|path| { + ( + (*path).to_string(), + crate::column::DynamicColumn::try_new( + &[0], + vec![DynamicChild::Typed { + ch_type: ChType::UInt8, + values: Column::UInt8(PrimitiveColumn::new(vec![79])), + }], + ) + .unwrap(), + ) + }) + .collect(); + let column = Column::Json(JsonColumn::structured(StructuredJson::from_parts( + vec![("a".into(), Column::Int64(PrimitiveColumn::new(vec![13])))], + dynamic, + vec![0, shared_names.len() as i64], + paths, + values, + 1, + ))); + let mut encoded = vec![0xab]; + assert!(matches!(encode_binary_value(&mut encoded, &ty, &column, 0), + Err(BinaryValueError::Invalid(reason)) if reason.contains("duplicate JSON path"))); + assert_eq!(encoded, [0xab]); + } + let mut bytes = vec![2]; + for _ in 0..2 { + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &ChType::UInt8); + bytes.push(13); + } + assert!(matches!(decode_binary_value(&parse("JSON"), &bytes), + Err(BinaryValueError::Invalid(reason)) if reason.contains("duplicate JSON path"))); + } + + #[test] + fn geo_and_json_geo_values_roundtrip_at_the_type_depth_limit() { + for leaf in [ + "Point", + "JSON(a Point)", + "SimpleAggregateFunction(any, Point)", + "Nested(a Point)", + ] { + let mut ty = parse(leaf); + let leaf_depth = match leaf { + "Point" => 1, + "Nested(a Point)" => 3, + _ => 2, + }; + let mut bytes = vec![1; MAX_TYPE_DEPTH - leaf_depth]; + for _ in leaf_depth..MAX_TYPE_DEPTH { + ty = ChType::Array(Box::new(ty)); + } + if leaf.starts_with("JSON") { + bytes.push(1); + write_string(&mut bytes, b"a"); + } else if leaf.starts_with("Nested") { + bytes.push(1); + } + bytes.extend_from_slice(&1.25f64.to_le_bytes()); + bytes.extend_from_slice(&(-3.5f64).to_le_bytes()); + let column = decode_binary_value(&ty, &bytes).unwrap(); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes, "{leaf}"); + } + } + + #[test] + fn json_qbit_path_roundtrips_at_the_type_depth_limit() { + let mut ty = parse("JSON"); + let mut bytes = vec![1; MAX_TYPE_DEPTH - 1]; + for _ in 1..MAX_TYPE_DEPTH { + ty = ChType::Array(Box::new(ty)); + } + bytes.push(1); + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &parse("QBit(Float64, 1)")); + bytes.push(1); + bytes.extend_from_slice(&1.25f64.to_le_bytes()); + let column = decode_binary_value(&ty, &bytes).unwrap(); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes); + } + + #[test] + fn descriptor_node_counts_follow_binary_writer() { + for (name, nodes) in [ + ("UInt8", 1), + ("Array(Nullable(UInt8))", 3), + ("Tuple(UInt8, Array(UInt8))", 4), + ("Map(String, UInt8)", 3), + ("Nested(x UInt8, y Array(UInt8))", 4), + ("Point", 1), + ("SimpleAggregateFunction(any, Array(UInt8))", 1), + ("Array(SimpleAggregateFunction(any, Array(UInt8)))", 2), + ("QBit(Float64, 1)", 2), + ("Tuple(QBit(Float64, 1), UInt8)", 4), + ("JSON(x Nullable(UInt8))", 3), + ("Variant(String, UInt8)", 3), + ] { + let ty = parse(name); + assert_eq!(check_type(&ty, None, 0).unwrap().descriptor_nodes, nodes); + let tuple = ChType::Tuple(vec![(None, ty); MAX_BINARY_TYPE_COMPLEXITY / nodes]); + let check = check_type(&tuple, None, 0).unwrap(); + let mut bytes = Vec::new(); + write_binary_type(&mut bytes, &tuple); + assert_eq!( + read_binary_type_prefix(&bytes).is_ok(), + check.check_descriptor_complexity().is_ok(), + "{name}" + ); + } + } + + #[test] + fn json_dynamic_child_descriptor_complexity_matches_decode() { + for count in [2000, 2001] { + let child_type = ChType::Tuple(vec![(None, ChType::UInt8); count]); + let child_values = Column::Tuple(TupleColumn::new( + vec![Column::UInt8(PrimitiveColumn::new(vec![79])); count], + 1, + )); + let dynamic = crate::column::DynamicColumn::try_new( + &[0], + vec![DynamicChild::Typed { + ch_type: child_type.clone(), + values: child_values, + }], + ) + .unwrap(); + let column = Column::Json(JsonColumn::structured(StructuredJson::from_parts( + vec![], + vec![("x".into(), dynamic)], + vec![0, 0], + Utf8Column::new(vec![0], vec![]), + Utf8Column::new(vec![0], vec![]), + 1, + ))); + let ty = parse("JSON"); + let mut encoded = vec![0xab]; + let result = encode_binary_value(&mut encoded, &ty, &column, 0); + if count == 2000 { + result.unwrap(); + let decoded = decode_binary_value(&ty, &encoded[1..]).unwrap(); + let Column::Json(json) = &decoded else { + panic!("expected JSON") + }; + let JsonBody::Structured(body) = &json.body else { + panic!("expected structured JSON") + }; + let cell = body.shared_values.value(0); + let (decoded_type, prefix_len) = read_binary_type_prefix(cell).unwrap(); + assert_eq!(decoded_type, child_type); + let Column::Tuple(tuple) = + decode_binary_value(&decoded_type, &cell[prefix_len..]).unwrap() + else { + panic!("expected Tuple") + }; + assert_eq!(tuple.fields.len(), count); + assert!(tuple + .fields + .iter() + .all(|field| matches!(field, Column::UInt8(values) if values.values == [79]))); + let mut reencoded = vec![0xab]; + encode_binary_value(&mut reencoded, &ty, &decoded, 0).unwrap(); + assert_eq!(reencoded, encoded); + } else { + assert!(matches!(result, + Err(BinaryValueError::Invalid(reason)) if reason.contains("complexity"))); + assert_eq!(encoded, [0xab]); + } + } + } + + #[test] + fn json_preserves_tagged_nothing_shared_values() { + let ty = parse("JSON"); + let bytes = [1, 1, b'x', 0]; + let column = decode_binary_value(&ty, &bytes).unwrap(); + let Column::Json(json) = &column else { + panic!("expected JSON") + }; + let JsonBody::Structured(body) = &json.body else { + panic!("expected structured JSON") + }; + assert_eq!(body.shared_offsets, [0, 1]); + assert_eq!(body.shared_paths.value(0), b"x"); + assert_eq!(body.shared_values.value(0), [0]); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes); + assert!(matches!( + decode_binary_value(&ChType::Nothing, &[]), + Err(BinaryValueError::Invalid(reason)) if reason == "non-null Nothing value" + )); + } + + #[test] + fn json_shared_paths_are_sorted_with_values() { + let ty = parse("JSON"); + let mut bytes = vec![4]; + for (path, value) in [(b"z", 13u8), (&[255], 17), (b"a", 79), (&[128], 31)] { + write_string(&mut bytes, path); + write_binary_type(&mut bytes, &ChType::UInt8); + bytes.push(value); + } + let Column::Json(json) = decode_binary_value(&ty, &bytes).unwrap() else { + panic!("expected JSON") + }; + let JsonBody::Structured(body) = json.body else { + panic!("expected structured JSON") + }; + assert_eq!(body.shared_paths.value(0), b"a"); + assert_eq!(body.shared_values.value(0), [1, 79]); + assert_eq!(body.shared_paths.value(1), b"z"); + assert_eq!(body.shared_values.value(1), [1, 13]); + assert_eq!(body.shared_paths.value(2), [128]); + assert_eq!(body.shared_values.value(2), [1, 31]); + assert_eq!(body.shared_paths.value(3), [255]); + assert_eq!(body.shared_values.value(3), [1, 17]); + let column = Column::Json(JsonColumn::structured(*body)); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(decode_binary_value(&ty, &encoded).unwrap(), column); + } + + #[test] + fn json_rejects_unsorted_or_duplicate_typed_metadata() { + for names in [["z", "a"], ["a", "a"]] { + let ty = ChType::Json { + max_dynamic_paths: 0, + max_dynamic_types: 0, + typed_paths: names + .iter() + .map(|name| ((*name).into(), ChType::UInt8)) + .collect(), + skip_paths: Vec::new(), + skip_regexps: Vec::new(), + }; + assert!(matches!( + decode_binary_value(&ty, &[0]), + Err(BinaryValueError::Invalid(_)) + )); + let column = Column::Json(JsonColumn::structured(StructuredJson::from_parts( + names + .iter() + .map(|name| { + ( + (*name).into(), + Column::UInt8(PrimitiveColumn::new(vec![13])), + ) + }) + .collect(), + Vec::new(), + vec![0, 0], + Utf8Column::new(vec![0], Vec::new()), + Utf8Column::new(vec![0], Vec::new()), + 1, + ))); + let mut encoded = vec![0xab]; + assert!(matches!( + encode_binary_value(&mut encoded, &parse("JSON(a UInt8, z UInt8)"), &column, 0), + Err(BinaryValueError::Invalid(_)) + )); + assert_eq!(encoded, [0xab]); + } + } + + #[test] + fn json_repeated_values_do_not_repeat_type_walks() { + let fields = (0..990) + .map(|index| (format!("v{index:03}"), ChType::UInt8)) + .collect(); + let nested = ChType::Array(Box::new(ChType::Nested(fields))); + let mut fields = vec![(None, ChType::Tuple(Vec::new())); 990]; + fields.push((None, ChType::Int8)); + let tuple = ChType::Array(Box::new(ChType::Tuple(fields))); + for (ty, value) in [(nested, 0), (tuple, 13)] { + let mut bytes = vec![1]; + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &ty); + write_varint(&mut bytes, 79); + bytes.extend(std::iter::repeat(value).take(79)); + SKIP_STEPS.with(|steps| steps.set(0)); + decode_binary_value(&parse("JSON"), &bytes).unwrap(); + let steps = SKIP_STEPS.with(|steps| steps.get()); + assert!(steps <= 79 + 2, "{steps} repeated type steps"); + } + } + + #[test] + fn json_caps_zero_width_tuple_shared_value_counts() { + let ty = parse("JSON"); + for (count, valid) in [ + (MAX_ZERO_WIDTH_ELEMENTS, true), + (MAX_ZERO_WIDTH_ELEMENTS + 1, false), + (usize::MAX, false), + ] { + let mut bytes = vec![1]; + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &parse("Array(Tuple())")); + write_varint(&mut bytes, count as u64); + match decode_binary_value(&ty, &bytes) { + Ok(column) => { + assert!(valid); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes); + } + Err(BinaryValueError::Invalid(reason)) => { + assert!(!valid, "{reason}"); + assert!(reason.contains("Array element count")); + } + Err(error) => panic!("{error}"), + } + } + let mut bytes = vec![1]; + write_string(&mut bytes, b"x"); + write_binary_type(&mut bytes, &parse("Array(Array(Tuple()))")); + write_varint(&mut bytes, 13); + for _ in 0..13 { + write_varint(&mut bytes, MAX_ZERO_WIDTH_ELEMENTS as u64); + } + let column = decode_binary_value(&ty, &bytes).unwrap(); + let mut encoded = Vec::new(); + encode_binary_value(&mut encoded, &ty, &column, 0).unwrap(); + assert_eq!(encoded, bytes); + } + #[test] fn rejects_truncated_and_trailing_payloads() { assert!(matches!( @@ -2256,17 +3443,17 @@ mod tests { ( "Array(Tuple(Nothing))", array(vec![0, 3], tuple(1, vec![nothing()])), - "unsupported binary value type: Nothing", + "non-null Nothing value", ), ( "Array(Tuple(Tuple(), Nothing))", array(vec![0, 3], tuple(1, vec![tuple(3, vec![]), nothing()])), - "unsupported binary value type: Nothing", + "non-null Nothing value", ), ( "Map(Tuple(), Tuple(Nothing))", map(vec![0, 3], tuple(1, vec![]), tuple(3, vec![nothing()]), 3), - "unsupported binary value type: Nothing", + "non-null Nothing value", ), ]; for (name, column, expected) in cases { @@ -2400,7 +3587,7 @@ mod tests { bytes.push(0x00); assert!(matches!( decode_binary_value(&parse("Array(Nothing)"), &bytes), - Err(BinaryValueError::Unsupported(_)) + Err(BinaryValueError::Invalid(reason)) if reason == "non-null Nothing value" )); let bytes = varint(0); assert!(decode_binary_value(&parse("Array(Nothing)"), &bytes).is_ok()); @@ -2423,9 +3610,6 @@ mod tests { assert_eq!(parsed, ch_type); assert_eq!(consumed, bytes.len() - 4); } - // A JSON descriptor now parses (type_binary supports tag 0x30), so the - // prefix reader accepts it, but a JSON value is never materialized as a - // shared cell: decode_binary_value stays Unsupported rather than panicking. let json_type = parse("JSON"); let mut json_bytes = Vec::new(); write_binary_type(&mut json_bytes, &json_type); @@ -2434,7 +3618,7 @@ mod tests { assert_eq!(consumed, json_bytes.len()); assert!(matches!( decode_binary_value(&json_type, &[]), - Err(BinaryValueError::Unsupported(_)) + Err(BinaryValueError::Invalid(_)) )); // A truncated descriptor is invalid. assert!(matches!( @@ -2650,6 +3834,55 @@ mod tests { } } + #[test] + fn skip_plan_consumption_matches_supported_value_decoding() { + let mut cases = value_cases(); + cases.extend([ + ("JSON", vec![0]), + ("JSON(a Int64)", vec![0]), + ("JSON", vec![2, 1, b'z', 0, 1, b'a', 1, 79]), + ]); + for (name, bytes) in cases { + let ty = parse(name); + let mut array = varint(2); + array.extend_from_slice(&bytes); + array.extend_from_slice(&bytes); + let mut tuple = bytes.clone(); + tuple.push(79); + let mut map = vec![1, 1, b'k']; + map.extend_from_slice(&bytes); + for (ty, bytes) in [ + (ty.clone(), bytes), + (ChType::Array(Box::new(ty.clone())), array), + ( + ChType::Tuple(vec![(None, ty.clone()), (None, ChType::UInt8)]), + tuple, + ), + (ChType::Map(Box::new(ChType::String), Box::new(ty)), map), + ] { + decode_binary_value(&ty, &bytes).unwrap_or_else(|error| panic!("{ty}: {error}")); + let plan = SkipPlan::new(&ty, 0).unwrap(); + let mut input = bytes.clone(); + input.extend_from_slice(&[0xab, 0xcd]); + let mut reader = ByteReader::new(&input); + plan.skip(&mut reader) + .unwrap_or_else(|error| panic!("{ty}: {error}")); + assert_eq!(reader.position(), bytes.len(), "{ty}"); + assert_eq!(reader.remaining_slice(), [0xab, 0xcd], "{ty}"); + for end in 0..bytes.len() { + assert!( + decode_binary_value(&ty, &bytes[..end]).is_err(), + "{ty}: {end}" + ); + assert!( + plan.skip(&mut ByteReader::new(&bytes[..end])).is_err(), + "{ty}: {end}" + ); + } + } + } + } + #[test] fn encode_reads_any_row_of_bulk_columns() { // LowCardinality(Nullable(String)) in its bulk Dictionary shape: index 0 @@ -2848,7 +4081,7 @@ mod tests { } for result in [value_result, prefix_result] { assert!(matches!(result, Err(BinaryValueError::Invalid(reason)) - if reason == "type nesting too deep")); + if reason == format!("nesting exceeds {MAX_TYPE_DEPTH}"))); } assert_eq!(buf, [0xaa]); } @@ -3023,7 +4256,7 @@ mod tests { // types. The encoder must reject without recursively formatting one. std::mem::forget(ch_type); assert!( - matches!(result, Err(BinaryValueError::Invalid(reason)) if reason == "type nesting too deep") + matches!(result, Err(BinaryValueError::Invalid(reason)) if reason == format!("nesting exceeds {MAX_TYPE_DEPTH}")) ); assert_eq!(buf, [0xaa]); } @@ -3038,7 +4271,7 @@ mod tests { let result = write_binary_type_prefix(&mut buf, &ch_type); std::mem::forget(ch_type); assert!(matches!(result, Err(BinaryValueError::Invalid(reason)) - if reason == "type nesting too deep")); + if reason == format!("nesting exceeds {MAX_TYPE_DEPTH}"))); assert_eq!(buf, [0xaa]); } @@ -3239,13 +4472,13 @@ mod tests { &ChType::Nothing, &Column::Nothing(NothingColumn::new(1)), 0, - true, + false, ); check( &parse("Nullable(Nothing)"), &Column::Nothing(NothingColumn::new_nullable(1, Bitmap::all_valid(1))), 0, - true, + false, ); } } diff --git a/src/native/type_binary.rs b/src/native/type_binary.rs index 48c2874..a497b91 100644 --- a/src/native/type_binary.rs +++ b/src/native/type_binary.rs @@ -20,7 +20,8 @@ use crate::schema::{ JSON_MAX_TYPED_PATHS, QBIT_MAX_DIMENSION, }; -const MAX_BINARY_TYPE_COMPLEXITY: usize = 1_000; +// Crate policy: a JSON root plus 1000 Nullable scalar typed paths. +pub(crate) const MAX_BINARY_TYPE_COMPLEXITY: usize = 2 * JSON_MAX_TYPED_PATHS + 1; const MAX_BINARY_TYPE_LIST: usize = 1_000_000; #[derive(Debug)] @@ -1178,4 +1179,49 @@ mod tests { Err(BinaryTypeError::Invalid(_)) )); } + + #[test] + fn json_maximum_typed_paths_fit_the_descriptor_budget() { + for value_type in ["Int64", "Nullable(Int64)"] { + let paths = (0..JSON_MAX_TYPED_PATHS) + .map(|index| format!("p{index:04} {value_type}")) + .collect::>() + .join(", "); + let json = parse_ch_type(&format!("JSON({paths})")).unwrap(); + let mut bytes = Vec::new(); + write_binary_type(&mut bytes, &json); + assert_eq!( + read_binary_type(&mut ByteReader::new(&bytes)).unwrap(), + json + ); + let mut scanner = BinaryTypeScanner::new(); + let mut consumed = 0; + for end in (1..bytes.len()) + .step_by(79) + .chain(std::iter::once(bytes.len())) + { + let mut reader = ByteReader::new(&bytes[consumed..end]); + let result = scanner.scan(&mut reader); + if end == bytes.len() { + result.unwrap(); + } else { + assert!( + matches!(result, Err(BinaryTypeError::Io(error)) if error.kind() == io::ErrorKind::UnexpectedEof) + ); + } + consumed += reader.position(); + } + assert_eq!(consumed, bytes.len()); + if value_type == "Nullable(Int64)" { + let mut oversized = Vec::new(); + write_binary_type(&mut oversized, &ChType::Array(Box::new(json))); + assert!(matches!(read_binary_type(&mut ByteReader::new(&oversized)), + Err(BinaryTypeError::Invalid(reason)) if reason.contains("complexity"))); + assert!( + matches!(BinaryTypeScanner::new().scan(&mut ByteReader::new(&oversized)), + Err(BinaryTypeError::Invalid(reason)) if reason.contains("complexity")) + ); + } + } + } } diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 4ded050..f8ea330 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -21,6 +21,13 @@ cross-checked against the matching `.server-ref` tag `v26.8.1.2041-lts`. | `json_string_rev0.native` | HTTP `FORMAT Native`, `output_format_native_write_json_as_string=1` | `0` | `01 04 01 6a 26 4a 53 4f 4e 28 6d 61 78 5f 64 79` | | `json_flattened_rev0.native` | HTTP `FORMAT Native`, `output_format_native_use_flattened_dynamic_and_json_serialization=1` | `0` | `01 04 01 6a 26 4a 53 4f 4e 28 6d 61 78 5f 64 79` | | `zero_width_dynamic_rev0.native` | HTTP `FORMAT Native` | `0` | `02 01 01 76 14 44 79 6e 61 6d 69 63 28 6d 61 78` | +| `json_shared_rev0.native` | HTTP `FORMAT Native` | `0` | `09 04 01 6a 14 44 79 6e 61 6d 69 63 28 6d 61 78` | +| `json_shared_rev54460.native` | Same query with `client_protocol_version=54460` | `54460` | `01 00 02 ff ff ff ff 00 09 04 01 6a 14 44 79 6e` | +| `json_shared_rev54485.native` | Same query with `client_protocol_version=54485` | `54485` | `01 00 02 ff ff ff ff 03 00 00 09 04 01 6a 14 44` | +| `json_multipart_rev0.native` | HTTP `FORMAT Native` from two MergeTree parts | `0` | `02 01 02 69 64 05 55 49 6e 74 38 00 01 64 07 44` | +| `json_multipart_rev54460.native` | Same table with `client_protocol_version=54460` | `54460` | `01 00 02 ff ff ff ff 00 02 01 02 69 64 05 55 49` | +| `json_multipart_rev54485.native` | Same table with `client_protocol_version=54485` | `54485` | `01 00 02 ff ff ff ff 03 00 00 02 01 02 69 64 05` | +| `json_mixed_rows_rev0.native` | HTTP `FORMAT Native`, mixed JSON runtime children | `0` | `01 04 01 6a 04 4a 53 4f 4e 00 00 00 00 00 00 00` | The revision-54485 fixture starts with the 10-byte `BlockInfo` preamble: `01 00 02 ff ff ff ff 03 00 00`. @@ -101,6 +108,43 @@ server emit an empty response, not a schema-bearing zero-row Native block. The decoder's zero-row block behavior remains covered by unit tests in `src/native/decode/tests/`. +## JSON values in Dynamic shared cells + +The `json_shared` and `json_multipart` fixtures were captured on 2026-11-17 +from ClickHouse `26.8.1.2041`, matched to source tag `v26.8.1.2041-lts`. +Regenerate them through `scripts/gen_fixtures.sh` with +`CLICKHOUSE_CONNECT_TEST_HOST=127.0.0.1` and `no_proxy=127.0.0.1`. +Revision 54460 uses Dynamic V1 and revision 54485 uses V2. The JSON +single-value payload uses the same path-count layout in both versions. + +`json_shared` contains four rows and nine columns. It covers JSON at the +root and inside Array, Tuple, and Map values, parameterized JSON with two +typed paths and skip lists, empty objects, null and absent paths, a +heterogeneous array, and a typed path whose own type is JSON. Standalone +typed and nested JSON columns provide comparison values for their copies +inside Dynamic. The tests decode shared cells through the public binary +value helpers and check the values, including `n.x = 13` and array element +`k = 1`. The first root JSON row has three untyped paths. Its helper output +must retain the path/value pairs in sorted order `[a, n.x, z]`. The +heterogeneous array check asserts its descriptor and every tagged element. + +`json_multipart` reads a plain Dynamic column from two MergeTree parts, +written in separate INSERT requests with merges stopped. One row contains +JSON and the other Array(JSON). The capture script creates a uniquely +named table and drops it on completion or failure. These captures also +exercise separate Native blocks. + +A JSON row's typed-path order comes from an unordered server map, and its +binary descriptor path lists are canonicalized by the core. JSON re-encoding +tests therefore check decoded values rather than requiring identical path +ordering in the bytes. + +`json_mixed_rows_rev0.native` contains one JSON column and four rows: an +empty object, a heterogeneous array path, a scalar path, and an empty array +path. The heterogeneous row reaches an unsupported Dynamic leaf in the +single-value writer. The other rows encode successfully, including the empty +Array(Dynamic), without inspecting the unrelated row's runtime values. + ## Refreshing Run: diff --git a/tests/fixtures/json_mixed_rows_rev0.native b/tests/fixtures/json_mixed_rows_rev0.native new file mode 100644 index 0000000000000000000000000000000000000000..2d28da258624022b5f65eadcc37786cd263c1b8f GIT binary patch literal 171 zcmZQ%Va#Ij3ikJ7fC5HF=G@E*MktSoiO;d9D6vw*r7|xuH#1q2)ibZe%ml6&t_&j1 m7F<%4nU@Ze{?EY3zzpTUXa=VLFvHnOi&Kl@4S8Y83D5wmp%O{} literal 0 HcmV?d00001 diff --git a/tests/fixtures/json_multipart_rev0.native b/tests/fixtures/json_multipart_rev0.native new file mode 100644 index 0000000000000000000000000000000000000000..380f07f6ac93956a268bad6f2f350bc701f645a9 GIT binary patch literal 128 zcmZQ#WXeoo4fV_`v0z|KVRxy_OU%toW@NwsVg?Ki911{TM&>-d3NBu#0w#ohM*RBa N;07=@JmgiMg4{42%pIK+J%lfkOc(%*dRlSHZ;# WRltbFBt`-z$-#|cWX$G**#!VJR~IAz literal 0 HcmV?d00001 diff --git a/tests/fixtures/json_multipart_rev54485.native b/tests/fixtures/json_multipart_rev54485.native new file mode 100644 index 0000000000000000000000000000000000000000..a18e2177ad2323e4d6a5e6dcbf96984ad3cc5715 GIT binary patch literal 150 zcmZQ%VEPXP%nS@nj7*s+tf8KHB^C?}j4A9cm3fJ|naK=H4Cp}2fT4jy0VvAIoTpd8 Z#S4{Z#A*~Hexu~zW-&5mb1_1V0sue97c2k( literal 0 HcmV?d00001 diff --git a/tests/fixtures/json_shared_rev0.native b/tests/fixtures/json_shared_rev0.native new file mode 100644 index 0000000000000000000000000000000000000000..790b2087a63046157ce4aa23cdfc499923df4788 GIT binary patch literal 1648 zcmc&!OH0E*5Z+0Q+F-$gRZn%nLt6||K|F}wEK&t6_TtI1Z5A8Nqa-P`p8V-8#V$q?WeObPflA+An$!ni#)GV;W2^P6@`E&`&y}H zxIeYa>QF*3nQNP90|~`@1}jpnC(0;|(a`@DVIrS-cg@X~ryQ7aPw1p8dD zN);EQMv{RQh16-Yc_p{3)C;T~c*8@_d(O>LQtt2IBQ-ihv~CA-7%3)<&~#ec&8B3}T&dh@sd literal 0 HcmV?d00001 diff --git a/tests/fixtures/json_shared_rev54460.native b/tests/fixtures/json_shared_rev54460.native new file mode 100644 index 0000000000000000000000000000000000000000..de0157420f9b50c9a36e9f7faffb8887b1e6899f GIT binary patch literal 1665 zcmc&!%}c{D6whnzbj5*!Q%^JCVH*~iARa_-4w-@uck$%N*05FEZnPb2p8U@>P4nUu z6+8UM<-NS`k6!}Bcov5gm*9TmY!>(>G{?IyD^k53zQ0_3CPG|1Bm6&e$my{;_a z(Y{u!8SYQ*lDZVpn7neECO(BKCF-wL5 zS`KCOYU2B>=kr=COpXrg4(-uS5_#dkp`FWCo4V0xECQ?BP4m2Y+osjKQQ#%jx5GL=bpQA0z@vJBn#a@r7ff_33&3K>-I4V z5+&C7=w)YTXXp8Cfhe!yQwhm#!PDOL!ex#%s5)%k3l>xEU)5_wAs>83oxJQ&ai9Xz z3yA?d>+M8T-IKYVPc~UJCQE4_br6~2J%y#o;-G|H6o9b~Nj#EU>dE`f+6@d#lpsyq z!y^R)Ll_v{6n^kPanfW|suX2LnVHYM-f5T^PT*IMwGb84z3sMJ5jkc|5fWt35GPbl zSuhe<8I;l1%(hwI=GCSfoS)SU+Na&Xv)rLUyLZhFHNDAHI8Ju=jhn_phgKdZjuli^ zu>>GkCjx*g4kl@dP$MJjD^d=oX0w2MSvl0K>|!n|wC9hQ$ITRv<+3gUE6^_jE9%@@$eiq${D|60BQF46bK literal 0 HcmV?d00001 diff --git a/tests/integration.rs b/tests/integration.rs index 2bc2c03..ef650bf 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -82,6 +82,257 @@ fixture_tests! { protocol_revision: 0, assert: assert_json_flattened, }, + json_multipart_rev0: { + file: "json_multipart_rev0.native", + protocol_revision: 0, + assert: assert_json_multipart, + }, + json_multipart_rev54460: { + file: "json_multipart_rev54460.native", + protocol_revision: 54460, + assert: assert_json_multipart, + }, + json_multipart_rev54485: { + file: "json_multipart_rev54485.native", + protocol_revision: 54485, + assert: assert_json_multipart, + }, + json_mixed_rows_rev0: { + file: "json_mixed_rows_rev0.native", + protocol_revision: 0, + assert: assert_json_mixed_rows, + }, + json_shared_rev0: { + file: "json_shared_rev0.native", + protocol_revision: 0, + assert: assert_json_shared, + }, + json_shared_rev54460: { + file: "json_shared_rev54460.native", + protocol_revision: 54460, + assert: assert_json_shared, + }, + json_shared_rev54485: { + file: "json_shared_rev54485.native", + protocol_revision: 54485, + assert: assert_json_shared, + }, +} + +fn decode_shared_row(column: &Column, row: usize) -> (ChType, Column) { + use ch_core_rs::native::binary_value::{ + decode_binary_value, encode_binary_value, read_binary_type_prefix, write_binary_type_prefix, + }; + let Column::Dynamic(dynamic) = column else { + panic!("expected Dynamic") + }; + let (id, offset) = dynamic.value_position(row).unwrap(); + let DynamicChild::Shared(cells) = &dynamic.children[id as usize] else { + panic!("expected SharedVariant row") + }; + let cell = cells.value(offset as usize); + let (ch_type, consumed) = read_binary_type_prefix(cell).unwrap(); + let column = decode_binary_value(&ch_type, &cell[consumed..]) + .unwrap_or_else(|error| panic!("{ch_type}: {error}")); + assert_eq!(column.len(), 1); + let mut encoded = Vec::new(); + write_binary_type_prefix(&mut encoded, &ch_type).unwrap(); + encode_binary_value(&mut encoded, &ch_type, &column, 0).unwrap(); + let (encoded_type, encoded_prefix) = read_binary_type_prefix(&encoded).unwrap(); + assert_eq!(encoded_type, ch_type); + assert_eq!( + decode_binary_value(&encoded_type, &encoded[encoded_prefix..]).unwrap(), + column + ); + (ch_type, column) +} + +fn assert_json_shared_int(column: &Column, row: usize, path: &str, expected: Option) { + use ch_core_rs::native::binary_value::{decode_binary_value, read_binary_type_prefix}; + let body = structured_json(column); + let start = body.shared_offsets[row] as usize; + let end = body.shared_offsets[row + 1] as usize; + let index = (start..end).find(|&index| body.shared_paths.value(index) == path.as_bytes()); + if let Some(expected) = expected { + let cell = body.shared_values.value(index.expect("missing JSON path")); + let (ty, consumed) = read_binary_type_prefix(cell).unwrap(); + assert_eq!(ty, ChType::Int64); + let Column::Int64(values) = decode_binary_value(&ty, &cell[consumed..]).unwrap() else { + panic!("expected Int64 path") + }; + assert_eq!(values.values, [expected]); + } else { + assert!(index.is_none(), "unexpected JSON path {path}"); + } +} + +fn assert_json_shared(batch: &ChunkedBatch) { + use ch_core_rs::native::binary_value::{decode_binary_value, encode_binary_value}; + assert_eq!(batch.num_rows(), 4); + let mut row_number = 0; + for chunk in &batch.chunks { + for row in 0..chunk.num_rows { + let n = row_number; + let (_, root) = decode_shared_row(chunk.column(0), row); + assert_json_shared_int(&root, 0, "n.x", [Some(13), None, None, Some(79)][n]); + if n == 0 { + assert_json_shared_int(&root, 0, "a", Some(17)); + assert_json_shared_int(&root, 0, "z", Some(79)); + assert_utf8_column(&structured_json(&root).shared_paths, &[b"a", b"n.x", b"z"]); + } + let (_, array) = decode_shared_row(chunk.column(1), row); + let Column::Array(array) = array else { + panic!("expected Array(JSON)") + }; + assert_eq!(array.offsets, [0, 2]); + assert_json_shared_int(&array.values, 0, "k", [Some(1), None, None, Some(79)][n]); + assert_eq!( + structured_json(&array.values).shared_offsets[2], + structured_json(&array.values).shared_offsets[1] + ); + let (_, tuple) = decode_shared_row(chunk.column(2), row); + let Column::Tuple(tuple) = tuple else { + panic!("expected Tuple") + }; + assert_json_shared_int(&tuple.fields[0], 0, "k", Some(79)); + let Column::UInt8(integer) = &tuple.fields[1] else { + panic!("expected UInt8") + }; + assert_eq!(integer.values, [17]); + let (_, map) = decode_shared_row(chunk.column(3), row); + let Column::Map(map) = map else { + panic!("expected Map") + }; + assert_eq!(map.offsets, [0, 1]); + let Column::Tuple(entries) = map.entries.as_ref() else { + panic!("expected entries") + }; + assert_utf8_values(&entries.fields[0], &[b"user_1"]); + assert_json_shared_int(&entries.fields[1], 0, "k", Some(13)); + let (ty, typed) = decode_shared_row(chunk.column(5), row); + let body = structured_json(&typed); + assert_eq!(body.typed.len(), 2); + assert_eq!(body.typed[0].0, "a"); + let Column::Int64(values) = &body.typed[0].1 else { + panic!("expected typed Int64") + }; + assert_eq!(values.values, [[13], [0], [0], [79]][n]); + assert_eq!( + values.validity.as_ref().unwrap().is_valid(0), + n == 0 || n == 3 + ); + assert_eq!(body.typed[1].0, "b"); + assert_utf8_values(&body.typed[1].1, &[if n == 0 { b"user_1" } else { b"" }]); + assert_json_shared_int(&typed, 0, "d", if n == 0 { Some(79) } else { None }); + assert_json_shared_int(&typed, 0, "later", if n == 3 { Some(13) } else { None }); + assert_json_shared_int(&typed, 0, "dropped", None); + assert_json_shared_int(&typed, 0, "omit_me", None); + let mut standalone = Vec::new(); + encode_binary_value(&mut standalone, &ty, chunk.column(4), row).unwrap(); + assert_eq!(decode_binary_value(&ty, &standalone).unwrap(), typed); + let (_, heterogeneous) = decode_shared_row(chunk.column(6), row); + let body = structured_json(&heterogeneous); + assert_eq!(body.shared_paths.value(0), b"mix"); + assert_heterogeneous_json_cell(body.shared_values.value(0)); + let (nested_type, nested) = decode_shared_row(chunk.column(8), row); + let body = structured_json(&nested); + assert_eq!(body.typed[0].0, "child"); + assert_json_shared_int(&body.typed[0].1, 0, "k", Some(13)); + let mut standalone = Vec::new(); + encode_binary_value(&mut standalone, &nested_type, chunk.column(7), row).unwrap(); + assert_eq!( + decode_binary_value(&nested_type, &standalone).unwrap(), + nested + ); + row_number += 1; + } + } +} + +fn assert_heterogeneous_json_cell(cell: &[u8]) { + use ch_core_rs::native::binary_value::{read_binary_type_prefix, write_binary_type_prefix}; + let array_type = ChType::Array(Box::new(ChType::Dynamic { + max_types: JSON_DEFAULT_MAX_DYNAMIC_TYPES, + })); + let (ty, _) = read_binary_type_prefix(cell).unwrap(); + assert_eq!(ty, array_type); + let mut expected = Vec::new(); + write_binary_type_prefix(&mut expected, &array_type).unwrap(); + expected.push(3); + write_binary_type_prefix(&mut expected, &ChType::Int64).unwrap(); + expected.extend_from_slice(&1i64.to_le_bytes()); + write_binary_type_prefix(&mut expected, &ChType::String).unwrap(); + expected.push(6); + expected.extend_from_slice(b"user_1"); + write_binary_type_prefix(&mut expected, &ChType::Nothing).unwrap(); + assert_eq!(cell, expected); +} + +fn assert_json_mixed_rows(batch: &ChunkedBatch) { + use ch_core_rs::native::binary_value::{ + decode_binary_value, encode_binary_value, BinaryValueError, + }; + assert_eq!(batch.num_rows(), 4); + let mut n = 0; + for chunk in &batch.chunks { + for row in 0..chunk.num_rows { + let mut encoded = vec![0xab]; + let result = encode_binary_value( + &mut encoded, + &json_type(JSON_DEFAULT_MAX_DYNAMIC_PATHS, vec![]), + chunk.column(0), + row, + ); + if n == 1 { + assert!(matches!(result, Err(BinaryValueError::Unsupported(_)))); + assert_eq!(encoded, [0xab]); + } else { + result.unwrap(); + let column = decode_binary_value( + &json_type(JSON_DEFAULT_MAX_DYNAMIC_PATHS, vec![]), + &encoded[1..], + ) + .unwrap(); + if n == 3 { + let body = structured_json(&column); + assert_eq!(body.shared_paths.value(0), b"mix"); + let cell = body.shared_values.value(0); + let (ty, consumed) = + ch_core_rs::native::binary_value::read_binary_type_prefix(cell).unwrap(); + assert_eq!( + ty, + ChType::Array(Box::new(ChType::Dynamic { + max_types: JSON_DEFAULT_MAX_DYNAMIC_TYPES, + })) + ); + assert_eq!(&cell[consumed..], [0]); + } else { + assert_json_shared_int(&column, 0, "mix", if n == 2 { Some(13) } else { None }); + } + } + n += 1; + } + } +} + +fn assert_json_multipart(batch: &ChunkedBatch) { + assert_eq!(batch.num_rows(), 2); + let mut n = 0; + for chunk in &batch.chunks { + for row in 0..chunk.num_rows { + let (_, value) = decode_shared_row(chunk.column(1), row); + if n == 0 { + assert_json_shared_int(&value, 0, "n.x", Some(13)); + } else { + let Column::Array(array) = value else { + panic!("expected Array(JSON)") + }; + assert_eq!(array.offsets, [0, 1]); + assert_json_shared_int(&array.values, 0, "k", Some(1)); + } + n += 1; + } + } } #[test] From a2463013d104d14f2caaf0bab683b0c806156d7b Mon Sep 17 00:00:00 2001 From: Joe S Date: Thu, 19 Nov 2026 13:54:25 -0800 Subject: [PATCH 2/2] Address review comments on JSON shared cells Stop over-charging null JSON defaults, check dynamic child types on encode, and list SkipPlan variants explicitly. Co-Authored-By: Claude Opus 5.5 --- src/native/binary_value.rs | 101 +++++++++++++++++++++++++++++++------ src/native/type_binary.rs | 16 +++--- 2 files changed, 96 insertions(+), 21 deletions(-) diff --git a/src/native/binary_value.rs b/src/native/binary_value.rs index da4991a..0b7ee57 100644 --- a/src/native/binary_value.rs +++ b/src/native/binary_value.rs @@ -44,7 +44,8 @@ use crate::column::{ }; use crate::native::protocol::MAX_TYPE_DEPTH; use crate::native::type_binary::{ - read_binary_type, write_binary_type, BinaryTypeError, MAX_BINARY_TYPE_COMPLEXITY, + check_type_construction, read_binary_type, write_binary_type, BinaryTypeError, + MAX_BINARY_TYPE_COMPLEXITY, }; use crate::native::varint::{write_varint, ByteReader}; use crate::schema::{ChType, QBitElementType, GEOMETRY_EXPANSION_DEPTH}; @@ -141,12 +142,16 @@ impl std::error::Error for BinaryValueError {} /// descriptor is `Invalid`. pub fn read_binary_type_prefix(bytes: &[u8]) -> Result<(ChType, usize), BinaryValueError> { let mut reader = ByteReader::new(bytes); - let ch_type = read_binary_type(&mut reader).map_err(|err| match err { + let ch_type = read_binary_type(&mut reader).map_err(type_error)?; + Ok((ch_type, reader.position())) +} + +fn type_error(err: BinaryTypeError) -> BinaryValueError { + match err { BinaryTypeError::Unsupported(reason) => BinaryValueError::Unsupported(reason), BinaryTypeError::Invalid(reason) => BinaryValueError::Invalid(reason), BinaryTypeError::Io(err) => BinaryValueError::from(err), - })?; - Ok((ch_type, reader.position())) + } } /// Decode exactly one `serializeBinary` value of `ch_type` from `bytes` into a @@ -632,7 +637,6 @@ impl ValueBuilder { let row_bytes = reader.remaining_slice(); let count = read_element_count(reader, false, "JSON path")?; seen.fill(false); - let mut missing_typed = typed.len(); path_order.clear(); for _ in 0..count { let path = read_json_path(reader)?; @@ -645,7 +649,6 @@ impl ValueBuilder { )); } seen[index] = true; - missing_typed -= 1; typed[index].1.append_value(reader, null_budget)?; } else { let start = reader.position(); @@ -669,7 +672,7 @@ impl ValueBuilder { append_utf8(paths, &row_bytes[path.clone()])?; append_utf8(values, &row_bytes[cell.clone()])?; } - null_budget.charge(missing_typed)?; + // Each missing default charges the budget at least 1. for ((_, builder), seen) in typed.iter_mut().zip(seen.iter()) { if !seen { builder.append_default(null_budget)?; @@ -833,8 +836,11 @@ impl ValueBuilder { } ValueBuilder::Json(json) => { let JsonValueBuilder { typed, offsets, .. } = json.as_mut(); - null_budget.charge(typed.len())?; - null_budget.charge(8)?; + // Every other default charges at least 1, so only an empty + // JSON needs its own charge. + if typed.is_empty() { + null_budget.charge(1)?; + } for (_, field) in typed { field.append_default(null_budget)?; } @@ -1023,11 +1029,7 @@ fn read_json_path<'a>(reader: &mut ByteReader<'a>) -> Result<&'a [u8], BinaryVal } fn read_value_type(reader: &mut ByteReader, depth: usize) -> Result { - let ty = read_binary_type(reader).map_err(|error| match error { - BinaryTypeError::Unsupported(reason) => BinaryValueError::Unsupported(reason), - BinaryTypeError::Invalid(reason) => BinaryValueError::Invalid(reason), - BinaryTypeError::Io(error) => error.into(), - })?; + let ty = read_binary_type(reader).map_err(type_error)?; check_type(&ty, None, depth).ok_or_else(too_deep)?; if ty == ChType::Nothing { // Dynamic NULL has only its Nothing descriptor. @@ -1161,7 +1163,13 @@ impl SkipPlan { )) } ChType::AggregateFunction { .. } => return Ok(Self::Unsupported(ch_type.to_string())), - _ => { + // Bad Decimal widths and FixedString(0). Aliases returned above. + ChType::Decimal { .. } + | ChType::FixedString(_) + | ChType::SimpleAggregateFunction { .. } + | ChType::Geo(_) + | ChType::Geometry + | ChType::Nested(_) => { return Err(BinaryValueError::Invalid( "invalid binary value type".into(), )) @@ -1934,6 +1942,7 @@ fn encode_json_value( if !check.matches { return Err(mismatch(ch_type)); } + check_type_construction(ch_type).map_err(type_error)?; write_binary_type(buf, ch_type); encode_value(buf, ch_type, values, offset, depth + 1)?; } @@ -2688,6 +2697,16 @@ mod tests { )); } + #[test] + fn json_null_default_fits_one_max_width_fixed_string() { + let ty = parse("Nullable(JSON(a FixedString(16777215)))"); + for bytes in [&[1][..], &[0, 0]] { + let column = + decode_binary_value(&ty, bytes).unwrap_or_else(|e| panic!("decode {bytes:?}: {e}")); + assert_eq!(column.len(), 1); + } + } + #[test] fn json_missing_zero_width_defaults_charge_the_budget() { for name in [ @@ -3069,6 +3088,41 @@ mod tests { } } + #[test] + fn json_dynamic_child_rejects_illegal_type_construction() { + let child_type = ChType::Nullable(Box::new(parse("Array(UInt8)"))); + let dynamic = crate::column::DynamicColumn::try_new( + &[0], + vec![DynamicChild::Typed { + ch_type: child_type.clone(), + values: Column::Array(ArrayColumn::new( + vec![0, 1], + Column::UInt8(PrimitiveColumn::new(vec![79])), + )), + }], + ) + .unwrap(); + let column = Column::Json(JsonColumn::structured(StructuredJson::from_parts( + vec![], + vec![("x".into(), dynamic)], + vec![0, 0], + Utf8Column::new(vec![0], vec![]), + Utf8Column::new(vec![0], vec![]), + 1, + ))); + let mut cell = Vec::new(); + write_binary_type(&mut cell, &child_type); + let decode_err = read_binary_type_prefix(&cell).unwrap_err(); + let mut buf = vec![0xab]; + let encode_err = encode_binary_value(&mut buf, &parse("JSON"), &column, 0).unwrap_err(); + assert!(matches!(encode_err, BinaryValueError::Invalid(_))); + assert_eq!(encode_err.to_string(), decode_err.to_string()); + assert!(encode_err + .to_string() + .contains("violates ClickHouse type-construction rules")); + assert_eq!(buf, [0xab]); + } + #[test] fn json_preserves_tagged_nothing_shared_values() { let ty = parse("JSON"); @@ -3337,6 +3391,23 @@ mod tests { } } + #[test] + fn json_typed_zero_width_runs_take_steps_per_type_node() { + let fields = 999; + let name = format!( + "JSON(a Array(Tuple({})))", + vec!["Tuple()"; fields].join(", ") + ); + let ch_type = parse(&name); + let bytes = [vec![1, 1, b'a'], varint(MAX_ZERO_WIDTH_ELEMENTS as u64)].concat(); + let (column, decode_steps) = with_visits(|| decode_binary_value(&ch_type, &bytes)); + let column = column.unwrap(); + let (encoded, encode_steps) = with_visits(|| encode_one(&ch_type, &column, 0)); + assert_eq!(encoded, bytes); + assert!(decode_steps <= 2 * fields, "{decode_steps} decode steps"); + assert!(encode_steps <= 2 * fields, "{encode_steps} encode steps"); + } + #[test] fn empty_zero_width_runs_take_constant_steps() { let n = 10_000; diff --git a/src/native/type_binary.rs b/src/native/type_binary.rs index a497b91..6fc714b 100644 --- a/src/native/type_binary.rs +++ b/src/native/type_binary.rs @@ -54,15 +54,19 @@ impl std::fmt::Display for BinaryTypeError { pub(crate) fn read_binary_type(reader: &mut ByteReader) -> Result { let mut complexity = 0usize; let ch_type = read_binary_type_inner(reader, 0, &mut complexity)?; + check_type_construction(&ch_type)?; + Ok(ch_type) +} - // Structural tags bypass the textual parser that normally enforces server - // constructor rules. Round-tripping the canonical spelling through that - // parser applies the same Nullable/container/Variant legality checks and - // rejects malformed binary headers before zero-row `empty_column` or a body - // decoder can reach an impossible arm. +/// Structural tags bypass the textual parser that normally enforces server +/// constructor rules. Round-tripping the canonical spelling through that +/// parser applies the same Nullable/container/Variant legality checks and +/// rejects malformed binary headers before zero-row `empty_column` or a body +/// decoder can reach an impossible arm. +pub(crate) fn check_type_construction(ch_type: &ChType) -> Result<(), BinaryTypeError> { let canonical = ch_type.to_string(); match parse_ch_type(&canonical) { - Some(parsed) if parsed == ch_type => Ok(ch_type), + Some(parsed) if parsed == *ch_type => Ok(()), _ => Err(BinaryTypeError::Invalid(format!( "type {canonical} violates ClickHouse type-construction rules" ))),